launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #29528
[Merge] ~artivis/lpcraft:feature/yaml-include into lpcraft:main
jeremie has proposed merging ~artivis/lpcraft:feature/yaml-include into lpcraft:main with ~artivis/lpcraft:feature/yaml-alias-filter as a prerequisite.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~artivis/lpcraft/+git/lpcraft/+merge/435460
[Do Not Merge Yet]
Add support for yaml inclusion.
This allows for including other yaml files to a configuration file using the 'include' tag located at the root. Included file paths can be relative or absolute. Since yaml anchors are lost during the inclusion, this MP also introduce the 'extends' tag that essentially does the work of mapping.
Hereafter is an example of what that looks like,
# .included.launchpad.yaml
.test:
series: bionic
architectures: [amd64]
# .launchpad.yaml
pipeline:
- test
include:
- .included.launchpad.yaml
jobs:
test-a:
series: focal
architectures: [amd64]
test-b:
extends: .test # maps '.test' nested entries
packages: [file] # can be further extended with new entries
[Do Not Merge Yet]
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~artivis/lpcraft:feature/yaml-include into lpcraft:main.
diff --git a/lpcraft/config.py b/lpcraft/config.py
index 3564d9c..1409cde 100644
--- a/lpcraft/config.py
+++ b/lpcraft/config.py
@@ -373,7 +373,7 @@ class Config(ModelConfigDefaults):
"""Filter out extension field."""
expanded_values = values.copy()
for k, v in values.items():
- if k.startswith("."):
+ if k.startswith(".") or k == "include":
expanded_values.pop(k)
return expanded_values
diff --git a/lpcraft/tests/test_config.py b/lpcraft/tests/test_config.py
index 8695520..224f3c1 100644
--- a/lpcraft/tests/test_config.py
+++ b/lpcraft/tests/test_config.py
@@ -16,6 +16,7 @@ from testtools.matchers import (
MatchesStructure,
)
+<<<<<<< lpcraft/tests/test_config.py
from lpcraft.config import (
LAUNCHPAD_PPA_BASE_URL,
Config,
@@ -25,6 +26,10 @@ from lpcraft.config import (
get_ppa_url_parts,
)
from lpcraft.errors import CommandError
+=======
+from lpcraft.config import Config, OutputDistributeEnum, PackageRepository
+from lpcraft.errors import CommandError, ConfigurationError
+>>>>>>> lpcraft/tests/test_config.py
class TestConfig(TestCase):
@@ -35,8 +40,8 @@ class TestConfig(TestCase):
# So switch to the created project directory.
os.chdir(self.tempdir)
- def create_config(self, text):
- path = self.tempdir / ".launchpad.yaml"
+ def create_config(self, text, filename=".launchpad.yaml"):
+ path = self.tempdir / filename
path.write_text(text)
return path
@@ -909,3 +914,205 @@ class TestConfig(TestCase):
),
),
)
+
+ def test_yaml_include(self):
+ # Include a config file given a relative path
+ self.create_config(
+ dedent(
+ """
+ .test:
+ series: bionic
+ architectures: [amd64]
+ """
+ ),
+ ".included.launchpad.yaml",
+ )
+
+ path = self.create_config(
+ dedent(
+ """
+ pipeline:
+ - test
+
+ include:
+ - .included.launchpad.yaml
+
+ jobs:
+ test-a:
+ series: focal
+ architectures: [amd64]
+
+ test-b:
+ extends: .test
+ packages: [file]
+ """
+ )
+ )
+
+ config = Config.load(path)
+
+ self.assertThat(
+ config,
+ MatchesStructure(
+ pipeline=Equals([["test"]]),
+ jobs=MatchesDict(
+ {
+ "test-a": MatchesListwise(
+ [
+ MatchesStructure.byEquality(
+ series="focal",
+ architectures=["amd64"],
+ )
+ ]
+ ),
+ "test-b": MatchesListwise(
+ [
+ MatchesStructure.byEquality(
+ series="bionic",
+ architectures=["amd64"],
+ packages=["file"],
+ )
+ ]
+ ),
+ }
+ ),
+ ),
+ )
+
+ def test_yaml_include_abs_path(self):
+ # Include a config file given an absolute path
+ tempdir = Path(self.useFixture(TempDir()).path)
+ included_config_path = tempdir / ".included.launchpad.yaml"
+ included_config_path.write_text(
+ dedent(
+ """
+ .test:
+ series: bionic
+ architectures: [amd64]
+ """
+ )
+ )
+
+ base_config_path = self.create_config(
+ dedent(
+ f"""
+ pipeline:
+ - test
+
+ include:
+ - '{included_config_path}'
+
+ jobs:
+ test-a:
+ series: focal
+ architectures: [amd64]
+
+ test-b:
+ extends: .test
+ packages: [file]
+ """
+ )
+ )
+
+ config = Config.load(base_config_path)
+
+ self.assertNotEqual(base_config_path.parent, included_config_path.parent)
+
+ self.assertThat(
+ config,
+ MatchesStructure(
+ pipeline=Equals([["test"]]),
+ jobs=MatchesDict(
+ {
+ "test-a": MatchesListwise(
+ [
+ MatchesStructure.byEquality(
+ series="focal",
+ architectures=["amd64"],
+ )
+ ]
+ ),
+ "test-b": MatchesListwise(
+ [
+ MatchesStructure.byEquality(
+ series="bionic",
+ architectures=["amd64"],
+ packages=["file"],
+ )
+ ]
+ ),
+ }
+ ),
+ ),
+ )
+
+ def test_yaml_bad_include(self):
+ # Include a config file that doesn't exists.
+ path = self.create_config(
+ dedent(
+ """
+ pipeline:
+ - test
+
+ include:
+ - bad_include.launchpad.yaml
+
+ jobs:
+ test:
+ series: focal
+ architectures: [amd64]
+ """
+ )
+ )
+
+ self.assertRaisesRegex(
+ ConfigurationError,
+ "Couldn't find config file 'bad_include.launchpad.yaml'.",
+ Config.load,
+ path,
+ )
+
+ def test_yaml_include_not_mapping(self):
+ # Include a config file that is not defining a mapping
+ self.create_config(
+ dedent(
+ """
+ - foo
+ """
+ ),
+ ".launchpad.included.yaml",
+ )
+
+ path = self.create_config(
+ dedent(
+ """
+ include:
+ - .launchpad.included.yaml
+ """
+ )
+ )
+
+ self.assertRaisesRegex(
+ ConfigurationError,
+ "Config file '.launchpad.included.yaml' does not define a mapping",
+ Config.load,
+ path,
+ )
+
+ def test_yaml_bad_extends(self):
+ # Extends a node that doesn't exists.
+ path = self.create_config(
+ dedent(
+ """
+ test:
+ extends: .test
+ """
+ )
+ )
+
+ self.assertRaisesRegex(
+ ConfigurationError,
+ "Couldn't find extension '.test'.",
+ Config.load,
+ path,
+ )
diff --git a/lpcraft/tests/test_utils.py b/lpcraft/tests/test_utils.py
index 52b42e3..b4252bc 100644
--- a/lpcraft/tests/test_utils.py
+++ b/lpcraft/tests/test_utils.py
@@ -11,7 +11,7 @@ from systemfixtures import FakeProcesses
from testtools import TestCase
from lpcraft.errors import ConfigurationError
-from lpcraft.utils import ask_user, get_host_architecture, load_yaml
+from lpcraft.utils import ask_user, find, get_host_architecture, load_yaml
class TestLoadYAML(TestCase):
@@ -66,6 +66,16 @@ class TestLoadYAML(TestCase):
path,
)
+ def test_find(self):
+ self.assertEqual(42, next(find({'a':42}, 'a')))
+ self.assertEqual(42, next(find({'b':41,'a':42}, 'a')))
+
+ self.assertEqual(42, next(find({'b':{'a':42}}, 'a')))
+
+ self.assertEqual(42, next(find([{'b':41,'a':42}], 'a')))
+ self.assertEqual(42, next(find([{'b':41},{'a':42}], 'a')))
+
+
class TestGetHostArchitecture(TestCase):
def setUp(self):
diff --git a/lpcraft/utils.py b/lpcraft/utils.py
index a369cbd..1aa346f 100644
--- a/lpcraft/utils.py
+++ b/lpcraft/utils.py
@@ -18,6 +18,71 @@ import yaml
from lpcraft.errors import ConfigurationError
+def find(d: Any, key: Any) -> Any:
+ """Recursively search in dict/list/tuple."""
+ if isinstance(d, dict):
+ if key in d:
+ yield d[key]
+ for v in d.values():
+ for i in find(v, key):
+ yield i
+ if isinstance(d, (list, tuple)):
+ for v in d:
+ for i in find(v, key):
+ yield i
+
+
+def expand_yaml_extends(loaded: Any, root: Dict[Any, Any]) -> None:
+ """Expand 'extends' entries.
+
+ extends entries are searched for throughout the entire configuration.
+ When found, their associated value (pseudo-alias) is then
+ searched for throughout the entire configuration.
+ The value associated to the pseudo-alias is copied
+ at the level the initial 'extends' key was found at.
+ Finally the 'extends' entry is entirely removed.
+ """
+ tag = "extends"
+ if isinstance(loaded, dict):
+ for _, v in loaded.items():
+ if isinstance(v, (dict, list, tuple)):
+ expand_yaml_extends(v, root)
+ if tag in loaded.keys():
+ v = loaded.pop(tag)
+
+ try:
+ val = next(find(root, v))
+ except StopIteration:
+ raise ConfigurationError(
+ f"Couldn't find extension {str(v)!r}."
+ )
+
+ loaded.update(val)
+ elif isinstance(loaded, (list, tuple)):
+ for v in loaded:
+ expand_yaml_extends(v, root)
+
+
+def load_yaml_includes(loaded: Dict[str, Any]) -> None:
+ """Load all configuration files listed as include."""
+ for include in loaded.get("include", []):
+ include_path = Path(include)
+ if not include_path.is_file():
+ raise ConfigurationError(
+ f"Couldn't find config file {str(include_path)!r}."
+ )
+ with include_path.open("rb") as f:
+
+ included = yaml.safe_load(f)
+
+ if not isinstance(included, dict):
+ raise ConfigurationError(
+ f"Config file {str(include_path)!r} does not define a mapping"
+ )
+
+ loaded.update(included)
+
+
def load_yaml(path: Path) -> Dict[Any, Any]:
"""Return the content of a YAML file."""
if not path.is_file():
@@ -25,10 +90,16 @@ def load_yaml(path: Path) -> Dict[Any, Any]:
try:
with path.open("rb") as f:
loaded = yaml.safe_load(f)
+
if not isinstance(loaded, dict):
raise ConfigurationError(
f"Config file {str(path)!r} does not define a mapping"
)
+
+ load_yaml_includes(loaded)
+
+ expand_yaml_extends(loaded, loaded)
+
return loaded
except (yaml.error.YAMLError, OSError) as e:
raise ConfigurationError(
Follow ups
