launchpad-reviewers team mailing list archive

[Quentin Debhi <mp+474016@xxxxxxxxxxxxxxxxxx>]

Quentin Debhi has proposed merging ~ruinedyourlife/launchpad:add-sourcecraft-yaml-parser into launchpad:master with ~ruinedyourlife/launchpad:add-build-upload-processing-for-craft-recipes as a prerequisite.

Commit message:
Add sourcecraft yaml parser

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~ruinedyourlife/launchpad/+git/launchpad/+merge/474016
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of ~ruinedyourlife/launchpad:add-sourcecraft-yaml-parser into launchpad:master.

diff --git a/lib/lp/crafts/adapters/__init__.py b/lib/lp/crafts/adapters/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/lib/lp/crafts/adapters/__init__.py
diff --git a/lib/lp/crafts/adapters/buildarch.py b/lib/lp/crafts/adapters/buildarch.py
new file mode 100644
index 0000000..3f1ffde
--- /dev/null
+++ b/lib/lp/crafts/adapters/buildarch.py
@@ -0,0 +1,127 @@
+# Copyright 2024 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+__all__ = [
+    "determine_instances_to_build",
+]
+
+from collections import OrderedDict
+
+
+class CraftBasesParserError(Exception):
+    """Base class for all exceptions in this module."""
+
+
+class MissingPropertyError(CraftBasesParserError):
+    """Error for when an expected property is not present in the YAML."""
+
+    def __init__(self, prop, msg=None):
+        if msg is None:
+            msg = f"Craft specification is missing the {prop!r} property"
+        super().__init__(msg)
+        self.property = prop
+
+
+class BadPropertyError(CraftBasesParserError):
+    """Error for when a YAML property is malformed in some way."""
+
+
+class CraftBase:
+    """A single base in sourcecraft.yaml."""
+
+    VALID_BASES = {"ubuntu@20.04", "ubuntu@22.04", "ubuntu@24.04", "bare"}
+    VALID_BUILD_BASES = VALID_BASES | {"devel"}
+
+    def __init__(self, base_string, is_build_base=False):
+        if is_build_base:
+            if base_string not in self.VALID_BUILD_BASES:
+                raise BadPropertyError(
+                    f"Invalid build-base: {base_string!r}. "
+                    f"Must be one of "
+                    f"{', '.join(sorted(self.VALID_BUILD_BASES))}"
+                )
+        else:
+            if base_string not in self.VALID_BASES:
+                raise BadPropertyError(
+                    f"Invalid base: {base_string!r}. "
+                    f"Must be one of {', '.join(sorted(self.VALID_BASES))}"
+                )
+
+        self.base_string = base_string
+        if base_string == "bare":
+            self.name = "bare"
+            self.channel = None
+        elif base_string == "devel":
+            raise BadPropertyError("The 'devel' base is not supported for now")
+        else:
+            self.name, self.channel = base_string.split("@")
+
+    def __eq__(self, other):
+        if not isinstance(other, CraftBase):
+            return NotImplemented
+        return self.name == other.name and self.channel == other.channel
+
+    def __hash__(self):
+        return hash((self.name, self.channel))
+
+    def __str__(self):
+        return f"{self.name}@{self.channel}"
+
+
+def determine_instances_to_build(
+    sourcecraft_data, supported_arches, default_distro_series
+):
+    """Return a list of instances to build based on sourcecraft.yaml.
+
+    :param sourcecraft_data: A parsed sourcecraft.yaml.
+    :param supported_arches: An ordered list of all `DistroArchSeries` that
+        we can create builds for. Note that these may span multiple
+        `DistroSeries`.
+    :param default_distro_series: The default `DistroSeries` to use if
+        sourcecraft.yaml does not explicitly declare a base.
+    :return: A list of `DistroArchSeries`.
+    """
+    base = sourcecraft_data.get("base")
+    if not base:
+        raise MissingPropertyError("base")
+
+    craft_base = CraftBase(base)
+
+    if craft_base.base_string == "bare":
+        build_base = sourcecraft_data.get("build-base")
+        if not build_base:
+            raise MissingPropertyError(
+                "build-base",
+                "When 'base' is 'bare', 'build-base' must be specified",
+            )
+        craft_base = CraftBase(build_base, is_build_base=True)
+    elif "build-base" in sourcecraft_data:
+        craft_base = CraftBase(
+            sourcecraft_data["build-base"], is_build_base=True
+        )
+
+    platforms = sourcecraft_data.get("platforms")
+    if not platforms:
+        raise MissingPropertyError("platforms")
+
+    instances = OrderedDict()
+    for platform, config in platforms.items():
+        build_on = config.get("build-on", [platform])
+        build_for = config.get("build-for", [platform])
+
+        if isinstance(build_on, str):
+            build_on = [build_on]
+        if isinstance(build_for, str):
+            build_for = [build_for]
+
+        for arch in build_on:
+            for das in supported_arches:
+                if (
+                    das.distroseries.distribution.name == craft_base.name
+                    and das.distroseries.version == craft_base.channel
+                    and das.architecturetag == arch
+                ):
+                    instances[das] = None
+                    break
+
+    return list(instances)
diff --git a/lib/lp/crafts/adapters/tests/__init__.py b/lib/lp/crafts/adapters/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/lib/lp/crafts/adapters/tests/__init__.py
diff --git a/lib/lp/crafts/adapters/tests/test_buildarch.py b/lib/lp/crafts/adapters/tests/test_buildarch.py
new file mode 100644
index 0000000..f64c47f
--- /dev/null
+++ b/lib/lp/crafts/adapters/tests/test_buildarch.py
@@ -0,0 +1,246 @@
+# Copyright 2024 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from functools import partial
+
+from testscenarios import WithScenarios, load_tests_apply_scenarios
+from testtools.matchers import (
+    Equals,
+    MatchesException,
+    MatchesListwise,
+    MatchesStructure,
+    Raises,
+)
+from zope.component import getUtility
+
+from lp.app.interfaces.launchpad import ILaunchpadCelebrities
+from lp.buildmaster.interfaces.processor import (
+    IProcessorSet,
+    ProcessorNotFound,
+)
+from lp.crafts.adapters.buildarch import (
+    BadPropertyError,
+    CraftBase,
+    MissingPropertyError,
+    determine_instances_to_build,
+)
+from lp.testing import TestCase, TestCaseWithFactory
+from lp.testing.layers import LaunchpadZopelessLayer
+
+
+class TestCraftBase(TestCase):
+    def test_valid_base(self):
+        base = CraftBase("ubuntu@22.04")
+        self.assertEqual("ubuntu", base.name)
+        self.assertEqual("22.04", base.channel)
+
+    def test_bare_base(self):
+        base = CraftBase("bare")
+        self.assertEqual("bare", base.name)
+        self.assertIsNone(base.channel)
+
+    def test_invalid_base(self):
+        self.assertRaises(BadPropertyError, CraftBase, "ubuntu@18.04")
+
+    def test_invalid_build_base(self):
+        self.assertRaises(
+            BadPropertyError, CraftBase, "ubuntu@18.04", is_build_base=True
+        )
+
+    def test_devel_as_base_not_allowed(self):
+        self.assertRaises(BadPropertyError, CraftBase, "devel")
+
+    def test_str(self):
+        base = CraftBase("ubuntu@22.04")
+        self.assertEqual("ubuntu@22.04", str(base))
+        bare_base = CraftBase("bare")
+        self.assertEqual("bare@None", str(bare_base))
+
+    def test_equality(self):
+        base1 = CraftBase("ubuntu@22.04")
+        base2 = CraftBase("ubuntu@22.04")
+        base3 = CraftBase("ubuntu@20.04")
+        self.assertEqual(base1, base2)
+        self.assertNotEqual(base1, base3)
+
+
+class TestDetermineInstancesToBuild(WithScenarios, TestCaseWithFactory):
+    layer = LaunchpadZopelessLayer
+
+    scenarios = [
+        (
+            "single platform",
+            {
+                "sourcecraft_data": {
+                    "base": "ubuntu@22.04",
+                    "platforms": {
+                        "amd64": {},
+                    },
+                },
+                "expected": [("22.04", "amd64")],
+            },
+        ),
+        (
+            "multiple platforms",
+            {
+                "sourcecraft_data": {
+                    "base": "ubuntu@22.04",
+                    "platforms": {
+                        "amd64": {},
+                        "arm64": {},
+                    },
+                },
+                "expected": [("22.04", "amd64"), ("22.04", "arm64")],
+            },
+        ),
+        (
+            "build-on specified",
+            {
+                "sourcecraft_data": {
+                    "base": "ubuntu@22.04",
+                    "platforms": {
+                        "amd64": {"build-on": ["amd64", "arm64"]},
+                    },
+                },
+                "expected": [("22.04", "amd64"), ("22.04", "arm64")],
+            },
+        ),
+        (
+            "build-for specified",
+            {
+                "sourcecraft_data": {
+                    "base": "ubuntu@22.04",
+                    "platforms": {
+                        "amd64": {"build-for": "amd64"},
+                        "arm64": {"build-for": "arm64"},
+                    },
+                },
+                "expected": [("22.04", "amd64"), ("22.04", "arm64")],
+            },
+        ),
+        (
+            "bare base",
+            {
+                "sourcecraft_data": {
+                    "base": "bare",
+                    "build-base": "ubuntu@20.04",
+                    "platforms": {
+                        "amd64": {},
+                    },
+                },
+                "expected": [("20.04", "amd64")],
+            },
+        ),
+        (
+            "missing base",
+            {
+                "sourcecraft_data": {
+                    "platforms": {
+                        "amd64": {},
+                    },
+                },
+                "expected_exception": MatchesException(
+                    MissingPropertyError,
+                    "Craft specification is missing the 'base' property",
+                ),
+            },
+        ),
+        (
+            "missing build-base for bare",
+            {
+                "sourcecraft_data": {
+                    "base": "bare",
+                    "platforms": {
+                        "amd64": {},
+                    },
+                },
+                "expected_exception": MatchesException(
+                    MissingPropertyError,
+                    "When 'base' is 'bare', 'build-base' must be specified",
+                ),
+            },
+        ),
+        (
+            "missing platforms",
+            {
+                "sourcecraft_data": {
+                    "base": "ubuntu@22.04",
+                },
+                "expected_exception": MatchesException(
+                    MissingPropertyError,
+                    "Craft specification is missing the 'platforms' property",
+                ),
+            },
+        ),
+        (
+            "devel build-base not supported",
+            {
+                "sourcecraft_data": {
+                    "base": "ubuntu@22.04",
+                    "build-base": "devel",
+                    "platforms": {
+                        "amd64": {},
+                    },
+                },
+                "expected_exception": MatchesException(
+                    BadPropertyError,
+                    "The 'devel' base is not supported for now",
+                ),
+            },
+        ),
+    ]
+
+    def test_determine_instances_to_build(self):
+        distro_serieses = [
+            self.factory.makeDistroSeries(
+                distribution=getUtility(ILaunchpadCelebrities).ubuntu,
+                version=version,
+            )
+            for version in ("24.04", "22.04", "20.04")
+        ]
+        dases = []
+        for arch_tag in ("amd64", "arm64"):
+            try:
+                processor = getUtility(IProcessorSet).getByName(arch_tag)
+            except ProcessorNotFound:
+                processor = self.factory.makeProcessor(
+                    name=arch_tag, supports_virtualized=True
+                )
+            for distro_series in distro_serieses:
+                dases.append(
+                    self.factory.makeDistroArchSeries(
+                        distroseries=distro_series,
+                        architecturetag=arch_tag,
+                        processor=processor,
+                    )
+                )
+
+        build_instances_factory = partial(
+            determine_instances_to_build,
+            self.sourcecraft_data,
+            dases,
+            distro_serieses[0],
+        )
+
+        if hasattr(self, "expected_exception"):
+            self.assertThat(
+                build_instances_factory, Raises(self.expected_exception)
+            )
+        else:
+            self.assertThat(
+                build_instances_factory(),
+                MatchesListwise(
+                    [
+                        MatchesStructure(
+                            distroseries=MatchesStructure.byEquality(
+                                version=version
+                            ),
+                            architecturetag=Equals(arch_tag),
+                        )
+                        for version, arch_tag in self.expected
+                    ]
+                ),
+            )
+
+
+load_tests = load_tests_apply_scenarios