launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #24689
[Merge] lp:~twom/launchpad-buildd/include-base-in-digests into lp:launchpad-buildd
Tom Wardill has proposed merging lp:~twom/launchpad-buildd/include-base-in-digests into lp:launchpad-buildd.
Commit message:
Include base OS information in digests.json
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~twom/launchpad-buildd/include-base-in-digests/+merge/383584
For the final push to a registry, we need to tag the image with the base OS that it is built on.
Grab this information from `/etc/os-release` inside the image, parse it and save it to an appropriate place in the digests.json for processing downstream in Launchpad.
This is a breaking change in the format of digests.json, so needs the equivalent LP branch first.
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~twom/launchpad-buildd/include-base-in-digests into lp:launchpad-buildd.
=== modified file 'debian/changelog'
--- debian/changelog 2020-04-09 10:42:50 +0000
+++ debian/changelog 2020-05-07 11:10:50 +0000
@@ -1,3 +1,9 @@
+launchpad-buildd (190) UNRELEASED; urgency=medium
+
+ * Include base in OCI digests
+
+ -- Tom Wardill <tom.wardill@xxxxxxxxxxxxx> Thu, 07 May 2020 11:21:58 +0100
+
launchpad-buildd (189) xenial; urgency=medium
* Fix closing tar files in OCI builds
=== modified file 'lpbuildd/oci.py'
--- lpbuildd/oci.py 2020-04-09 07:51:23 +0000
+++ lpbuildd/oci.py 2020-05-07 11:10:50 +0000
@@ -147,6 +147,29 @@
return digest_diff_map
+ def _extractBase(self):
+ """Extract the base from the os-release file"""
+ os_file = tempfile.NamedTemporaryFile()
+ try:
+ self.backend.copy_out(
+ '/home/buildd/os-release',
+ os_file.name)
+ id = ""
+ version_id = ""
+ with open(os_file.name, 'r') as os_file_handle:
+ contents = [x.strip() for x in os_file_handle.readlines()]
+ for line in contents:
+ line_split = line.split('=')
+ if line_split[0] == "ID":
+ id = line_split[1]
+ elif line_split[0] == "VERSION_ID":
+ version_id = line_split[1].replace("\"", "")
+ return "{}{}".format(id, version_id)
+ except Exception as e:
+ print("Error getting os-release from docker: {}".format(e))
+ return ""
+
+
def gatherResults(self):
"""Gather the results of the build and add them to the file cache."""
extract_path = tempfile.mkdtemp(prefix=self.name)
@@ -214,9 +237,12 @@
digest_maps = []
try:
for section in manifest:
- digest_maps.append(
- self._gatherManifestSection(section, extract_path,
- sha_directory))
+ digests = self._gatherManifestSection(
+ section, extract_path, sha_directory)
+ base = self._extractBase()
+ digest_maps.append({
+ "digest_mappings": digests,
+ "base": base})
digest_map_file = os.path.join(extract_path, 'digests.json')
with open(digest_map_file, 'w') as digest_map_fp:
json.dump(digest_maps, digest_map_fp)
=== modified file 'lpbuildd/target/build_oci.py'
--- lpbuildd/target/build_oci.py 2020-03-31 13:54:56 +0000
+++ lpbuildd/target/build_oci.py 2020-05-07 11:10:50 +0000
@@ -16,6 +16,7 @@
from lpbuildd.target.snapbuildproxy import SnapBuildProxyOperationMixin
from lpbuildd.target.snapstore import SnapStoreOperationMixin
from lpbuildd.target.vcs import VCSOperationMixin
+import subprocess
RETCODE_FAILURE_INSTALL = 200
@@ -125,6 +126,22 @@
args.append(self.buildd_path)
self.run_build_command(args)
+ # Copy os-release file out, if it's present
+ args = ["docker", "container", "create",
+ "--name", "os-release-extraction", self.args.name]
+ self.run_build_command(args)
+ # docker cp test_lsb_release:/usr/lib/os-release os-release
+ args = ["docker", "cp", "-L", "os-release-extraction:/etc/os-release",
+ "/home/buildd/os-release"]
+ try:
+ self.run_build_command(args)
+ logger.info("os-release file extracted")
+ except subprocess.CalledProcessError as e:
+ # The os-release file may not exist causing this to error,
+ # we don't care.
+ logger.info("os-release file extraction failed: {}".format(e))
+ pass
+
def run(self):
try:
self.install()
=== modified file 'lpbuildd/target/tests/test_build_oci.py'
--- lpbuildd/target/tests/test_build_oci.py 2020-03-31 14:12:59 +0000
+++ lpbuildd/target/tests/test_build_oci.py 2020-05-07 11:10:50 +0000
@@ -304,6 +304,14 @@
["docker", "build", "--no-cache", "--tag", "test-image",
"/home/buildd/test-image"],
cwd="/home/buildd/test-image"),
+ RanBuildCommand(
+ ["docker", "container", "create", "--name",
+ "os-release-extraction", "test-image"],
+ cwd="/home/buildd/test-image"),
+ RanBuildCommand(
+ ["docker", "cp", "-L", "os-release-extraction:/etc/os-release",
+ "/home/buildd/os-release"],
+ cwd="/home/buildd/test-image"),
]))
def test_build_with_file(self):
@@ -322,6 +330,14 @@
"--file", "build-aux/Dockerfile",
"/home/buildd/test-image"],
cwd="/home/buildd/test-image"),
+ RanBuildCommand(
+ ["docker", "container", "create", "--name",
+ "os-release-extraction", "test-image"],
+ cwd="/home/buildd/test-image"),
+ RanBuildCommand(
+ ["docker", "cp", "-L", "os-release-extraction:/etc/os-release",
+ "/home/buildd/os-release"],
+ cwd="/home/buildd/test-image"),
]))
def test_build_proxy(self):
@@ -341,6 +357,14 @@
"--build-arg", "https_proxy=http://proxy.example:3128/";,
"--tag", "test-image", "/home/buildd/test-image"],
cwd="/home/buildd/test-image"),
+ RanBuildCommand(
+ ["docker", "container", "create", "--name",
+ "os-release-extraction", "test-image"],
+ cwd="/home/buildd/test-image"),
+ RanBuildCommand(
+ ["docker", "cp", "-L", "os-release-extraction:/etc/os-release",
+ "/home/buildd/os-release"],
+ cwd="/home/buildd/test-image"),
]))
def test_run_succeeds(self):
@@ -362,6 +386,14 @@
["docker", "build", "--no-cache", "--tag", "test-image",
"/home/buildd/test-image"],
cwd="/home/buildd/test-image")),
+ AnyMatch(RanBuildCommand(
+ ["docker", "container", "create", "--name",
+ "os-release-extraction", "test-image"],
+ cwd="/home/buildd/test-image")),
+ AnyMatch(RanBuildCommand(
+ ["docker", "cp", "-L", "os-release-extraction:/etc/os-release",
+ "/home/buildd/os-release"],
+ cwd="/home/buildd/test-image")),
))
def test_run_install_fails(self):
=== modified file 'lpbuildd/tests/test_oci.py'
--- lpbuildd/tests/test_oci.py 2020-02-26 10:52:29 +0000
+++ lpbuildd/tests/test_oci.py 2020-05-07 11:10:50 +0000
@@ -135,6 +135,13 @@
'vfs/distribution/v2metadata-by-diffid/sha256/diff1',
b"""[{"Digest": "test_digest", "SourceRepository": "test"}]"""
)
+ self.buildmanager.backend.add_file(
+ '/home/buildd/os-release',
+ b'ID=ubuntu\n'
+ b'ID_LIKE=debian\n'
+ b'PRETTY_NAME="Ubuntu 16.04.6 LTS"\n'
+ b'VERSION_ID="16.04"\n'
+ )
# After building the package, reap processes.
yield self.buildmanager.iterate(0)
@@ -162,16 +169,18 @@
with open(cache_path, "rb") as f:
digests_contents = f.read()
digests_expected = [{
- "sha256:diff1": {
- "source": "test",
- "digest": "test_digest",
- "layer_id": "layer-1"
- },
- "sha256:diff2": {
- "source": "",
- "digest": "testsha",
- "layer_id": "layer-2"
- }
+ "digest_mappings": {
+ "sha256:diff1": {
+ "source": "test",
+ "digest": "test_digest",
+ "layer_id": "layer-1"
+ },
+ "sha256:diff2": {
+ "source": "",
+ "digest": "testsha",
+ "layer_id": "layer-2"
+ }},
+ "base": "ubuntu16.04"
}]
self.assertEqual(digests_contents, json.dumps(digests_expected))
# Control returns to the DebianBuildManager in the UMOUNT state.
@@ -245,16 +254,18 @@
with open(cache_path, "rb") as f:
digests_contents = f.read()
digests_expected = [{
- "sha256:diff1": {
- "source": "test",
- "digest": "test_digest",
- "layer_id": "layer-1"
- },
- "sha256:diff2": {
- "source": "",
- "digest": "testsha",
- "layer_id": "layer-2"
- }
+ "digest_mappings": {
+ "sha256:diff1": {
+ "source": "test",
+ "digest": "test_digest",
+ "layer_id": "layer-1"
+ },
+ "sha256:diff2": {
+ "source": "",
+ "digest": "testsha",
+ "layer_id": "layer-2"
+ }},
+ "base": ""
}]
self.assertEqual(digests_contents, json.dumps(digests_expected))
