launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #25532
[Merge] ~pappacena/launchpad-buildd:security-manifest-packages into launchpad-buildd:master
Thiago F. Pappacena has proposed merging ~pappacena/launchpad-buildd:security-manifest-packages into launchpad-buildd:master with ~pappacena/launchpad-buildd:security-manifest-build-metadata as a prerequisite.
Commit message:
Adding OS info and packages to OCI security manifest file
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~pappacena/launchpad-buildd/+git/launchpad-buildd/+merge/392623
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~pappacena/launchpad-buildd:security-manifest-packages into launchpad-buildd:master.
diff --git a/lpbuildd/target/build_oci.py b/lpbuildd/target/build_oci.py
index 679a12f..b49677c 100644
--- a/lpbuildd/target/build_oci.py
+++ b/lpbuildd/target/build_oci.py
@@ -9,6 +9,7 @@ from collections import OrderedDict
import json
import logging
import os.path
+import re
import sys
import tempfile
from textwrap import dedent
@@ -112,7 +113,8 @@ class BuildOCI(SnapBuildProxyOperationMixin, VCSOperationMixin,
# Add any proxy settings that are needed
self._add_docker_engine_proxy_settings()
deps.extend(self.vcs_deps)
- deps.extend(["docker.io"])
+ # Install dctrl-tools to extract installed packages using grep-dctrl.
+ deps.extend(["docker.io", "dctrl-tools"])
self.backend.run(["apt-get", "-y", "install"] + deps)
if self.args.backend in ("lxd", "fake"):
self.snap_store_set_proxy()
@@ -136,6 +138,48 @@ class BuildOCI(SnapBuildProxyOperationMixin, VCSOperationMixin,
revision_cmd, cwd=os.path.join("/home/buildd", self.args.name),
get_output=True).decode("UTF-8", "replace").strip()
+ def _getContainerPackageList(self):
+ tmp_file = "/tmp/dpkg-status"
+ self.run_build_command([
+ "docker", "cp", "-L",
+ "%s:/var/lib/dpkg/status" % self.args.name, tmp_file])
+ output = self.backend.run([
+ "grep-dctrl", "-s", "Package,Version,Source", "", tmp_file],
+ get_output=True).decode("UTF-8", "replace")
+ packages = []
+ empty_pkg_details = dict.fromkeys(["package", "version", "source"])
+ current_package = empty_pkg_details.copy()
+ for line in output.split("\n"):
+ if not line.strip():
+ if not all(i is None for i in current_package.values()):
+ packages.append(current_package)
+ current_package = empty_pkg_details.copy()
+ continue
+ k, v = line.split(":", 1)
+ current_package[k.lower().strip()] = v.strip()
+ if not all(i is None for i in current_package.values()):
+ packages.append(current_package)
+ return packages
+
+ def _getContainerOSRelease(self):
+ tmp_file = "/tmp/os-release"
+ self.run_build_command([
+ "docker", "cp", "-L",
+ "%s:/etc/os-release" % self.args.name, tmp_file])
+ content = self.backend.run(["cat", tmp_file], get_output=True)
+ os_release = {}
+ # Variable content might be enclosed by double-quote, single-quote
+ # or no quote at all. We accept everything.
+ content_expr = re.compile(r""""(.*)"|'(.*)'|(.*)""")
+ unquote = lambda string: [
+ i for i in content_expr.match(string).groups() if i is not None][0]
+ for line in content.decode("UTF-8", "replace").split("\n"):
+ if '=' not in line:
+ continue
+ key, value = line.strip().split("=", 1)
+ os_release[key] = unquote(value)
+ return os_release
+
def _getSecurityManifestContent(self):
try:
metadata = json.loads(self.args.metadata) or {}
@@ -147,14 +191,26 @@ class BuildOCI(SnapBuildProxyOperationMixin, VCSOperationMixin,
if i.get("email")]
try:
+ packages = self._getContainerPackageList()
+ except Exception as e:
+ logger.warning("Failed to get container package list: %s", e)
+ packages = []
+ try:
vcs_current_version = self._getCurrentVCSRevision()
except Exception as e:
logger.warning("Failed to get current VCS revision: %s" % e)
vcs_current_version = None
+ try:
+ os_release = self._getContainerOSRelease()
+ except Exception as e:
+ logger.warning("Failed to get /etc/os-release info: %s" % e)
+ os_release = {}
return {
"manifest-version": "1",
"name": self.args.name,
+ "os-release-id": os_release.get("ID"),
+ "os-release-version-id": os_release.get("VERSION_ID"),
"architectures": metadata.get("architectures") or [self.args.arch],
"publisher-emails": emails,
"image-info": {
@@ -170,7 +226,8 @@ class BuildOCI(SnapBuildProxyOperationMixin, VCSOperationMixin,
"source-subdir": self.args.build_path,
"source-build-file": self.args.build_file,
"source-build-args": self.args.build_arg
- }]
+ }],
+ "packages": packages
}
def createSecurityManifest(self):
@@ -182,6 +239,7 @@ class BuildOCI(SnapBuildProxyOperationMixin, VCSOperationMixin,
destination_path = self.security_manifest_target_path.lstrip(
os.path.sep)
destination = os.path.join(self.backend_tmp_fs_dir, destination_path)
+ logger.info("Security manifest: %s" % content)
with open(local_filename, 'w') as fd:
json.dump(content, fd, indent=2)
self.backend.copy_in(local_filename, destination)
diff --git a/lpbuildd/target/tests/test_build_oci.py b/lpbuildd/target/tests/test_build_oci.py
index 0572b4e..7d79678 100644
--- a/lpbuildd/target/tests/test_build_oci.py
+++ b/lpbuildd/target/tests/test_build_oci.py
@@ -5,6 +5,10 @@ __metaclass__ = type
import datetime
import json
+try:
+ from unittest import mock
+except ImportError:
+ import mock
import os.path
import stat
import subprocess
@@ -99,10 +103,53 @@ class TestBuildOCIManifestGeneration(TestCase):
"test-image"
]
build_oci = parse_args(args=args).operation
- build_oci.backend.run.result = b"a1b2c3d4e5f5\n"
+
+ # Expected build_oci.backend.run outputs.
+ commit_hash = b"a1b2c3d4e5f5"
+ grep_dctrl_output = dedent("""
+ Package: adduser
+ Version: 3.118
+
+ Package: apt
+ Version: 1.8.2.1
+
+ Package: util-linux
+ Version: 2.33.1
+
+ Package: zlib1g
+ Version: 1:1.2.11
+ Source: zlib
+ """).encode('utf8')
+
+ os_release_cat_output = dedent("""
+ NAME="Ubuntu"
+ VERSION="20.04.1 LTS (Focal Fossa)"
+ ID=ubuntu
+ ID_LIKE=debian
+ PRETTY_NAME="Ubuntu 20.04.1 LTS"
+ VERSION_ID="20.04"
+ HOME_URL="https://www.ubuntu.com/";
+ SUPPORT_URL="https://help.ubuntu.com/";
+ BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/";
+ PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy";
+ VERSION_CODENAME=focal
+ UBUNTU_CODENAME=focal
+ """).encode("utf8")
+
+ # Side effect for "docker cp...", "dgrep-dctrl" and "git rev-parse..."
+ build_oci.backend.run = mock.Mock(side_effect=[
+ # docker cp and dgrep-dctrl to get packages.
+ None, grep_dctrl_output,
+ # git rev-parse HEAD to get current revision.
+ commit_hash,
+ # docker cp and cat for container /etc/os-release.
+ None, os_release_cat_output])
+
self.assertEqual(build_oci._getSecurityManifestContent(), {
"manifest-version": "1",
"name": "test-image",
+ 'os-release-id': "ubuntu",
+ 'os-release-version-id': "20.04",
"architectures": ["amd64", "386"],
"publisher-emails": ["me@xxxxxxx", "someone@xxxxxxx"],
"image-info": {
@@ -118,7 +165,13 @@ class TestBuildOCIManifestGeneration(TestCase):
"source-build-file": "SomeDockerfile",
"source-commit": "a1b2c3d4e5f5",
"source-subdir": "docker/builder"
- }]
+ }],
+ "packages": [
+ {'package': 'adduser', 'source': None, 'version': '3.118'},
+ {'package': 'apt', 'source': None, 'version': '1.8.2.1'},
+ {'package': 'util-linux', 'source': None, 'version': '2.33.1'},
+ {'package': 'zlib1g', 'source': 'zlib', 'version': '1:1.2.11'}
+ ]
})
def test_getSecurityManifestContent_without_manifest(self):
@@ -130,10 +183,17 @@ class TestBuildOCIManifestGeneration(TestCase):
"--git-repository", "lp:git-repo", "--git-path", "refs/heads/main",
"test-image"
]
+
+ # Here we will not mock the package gathering nor os-release file
+ # reading in order to let it raise exception, so we end up with a
+ # manifest without packages.
build_oci = parse_args(args=args).operation
+
self.assertEqual(build_oci._getSecurityManifestContent(), {
"manifest-version": "1",
"name": "test-image",
+ 'os-release-id': None,
+ 'os-release-version-id': None,
"architectures": ["amd64"],
"publisher-emails": [],
"image-info": {
@@ -147,9 +207,38 @@ class TestBuildOCIManifestGeneration(TestCase):
"source-build-file": None,
"source-commit": None,
"source-subdir": "."
- }]
+ }],
+ "packages": []
})
+ def test_getContainerPackageList(self):
+ args = [
+ "build-oci",
+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+ "--git-repository", "lp:git-repo", "--git-path", "refs/heads/main",
+ "test-image"
+ ]
+ build_oci = parse_args(args=args).operation
+ build_oci.backend.run = mock.Mock(return_value=dedent("""
+ Package: adduser
+ Version: 3.118
+
+ Package: apt
+ Version: 1.8.2.1
+
+ Package: util-linux
+ Version: 2.33.1-0.1
+
+ Package: zlib1g
+ Version: 1:1.2.11
+ Source: zlib
+ """).encode("utf8"))
+ self.assertEqual([
+ {'package': 'adduser', 'source': None, 'version': '3.118'},
+ {'package': 'apt', 'source': None, 'version': '1.8.2.1'},
+ {'package': 'util-linux', 'source': None, 'version': '2.33.1-0.1'},
+ {'package': 'zlib1g', 'source': 'zlib', 'version': '1:1.2.11'}
+ ], build_oci._getContainerPackageList())
class TestBuildOCI(TestCase):
@@ -192,7 +281,7 @@ class TestBuildOCI(TestCase):
build_oci = parse_args(args=args).operation
build_oci.install()
self.assertThat(build_oci.backend.run.calls, MatchesListwise([
- RanAptGet("install", "bzr", "docker.io"),
+ RanAptGet("install", "bzr", "docker.io", "dctrl-tools"),
RanCommand(["systemctl", "restart", "docker"]),
RanCommand(["mkdir", "-p", "/home/buildd"]),
]))
@@ -206,7 +295,7 @@ class TestBuildOCI(TestCase):
build_oci = parse_args(args=args).operation
build_oci.install()
self.assertThat(build_oci.backend.run.calls, MatchesListwise([
- RanAptGet("install", "git", "docker.io"),
+ RanAptGet("install", "git", "docker.io", "dctrl-tools"),
RanCommand(["systemctl", "restart", "docker"]),
RanCommand(["mkdir", "-p", "/home/buildd"]),
]))
@@ -262,7 +351,8 @@ class TestBuildOCI(TestCase):
self.assertThat(build_oci.backend.run.calls, MatchesListwise([
RanCommand(
["mkdir", "-p", "/etc/systemd/system/docker.service.d"]),
- RanAptGet("install", "python3", "socat", "git", "docker.io"),
+ RanAptGet("install", "python3", "socat", "git", "docker.io",
+ "dctrl-tools"),
RanCommand(["systemctl", "restart", "docker"]),
RanCommand(["mkdir", "-p", "/home/buildd"]),
]))
@@ -377,8 +467,28 @@ class TestBuildOCI(TestCase):
['docker', 'create', '--name', 'test-image', 'test-image'],
cwd="/home/buildd/test-image"),
RanCommand(['mkdir', '-p', '/tmp/image-root-dir/.rocks']),
+
+ # Manifest building: packages discovery.
+ RanBuildCommand([
+ 'docker', 'cp', '-L',
+ 'test-image:/var/lib/dpkg/status', '/tmp/dpkg-status'],
+ cwd="/home/buildd/test-image"),
+ RanCommand([
+ 'grep-dctrl', '-s', 'Package,Version,Source', '',
+ '/tmp/dpkg-status'], get_output=True),
+
+ # Manifest building: get current revision number.
RanCommand(
rev_num_args, cwd="/home/buildd/test-image", get_output=True),
+
+ # Manifest building: os-release file.
+ RanBuildCommand([
+ 'docker', 'cp', '-L', 'test-image:/etc/os-release',
+ '/tmp/os-release'],
+ cwd="/home/buildd/test-image"),
+ RanCommand(['cat', '/tmp/os-release'], get_output=True),
+
+ # Filesystem injection and image commiting.
RanBuildCommand(
['docker', 'cp', '/tmp/image-root-dir/.', 'test-image:/'],
cwd="/home/buildd/test-image"),
@@ -505,7 +615,7 @@ class TestBuildOCI(TestCase):
build_oci.backend.run = FakeMethod()
self.assertEqual(0, build_oci.run())
self.assertThat(build_oci.backend.run.calls, MatchesAll(
- AnyMatch(RanAptGet("install", "bzr", "docker.io")),
+ AnyMatch(RanAptGet("install", "bzr", "docker.io", "dctrl-tools")),
AnyMatch(RanBuildCommand(
["bzr", "branch", "lp:foo", "test-image"],
cwd="/home/buildd")),
