launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #27777
[Merge] ~cjwatson/lpcraft:output into lpcraft:main
Colin Watson has proposed merging ~cjwatson/lpcraft:output into lpcraft:main with ~cjwatson/lpcraft:remove-managed-mode as a prerequisite.
Commit message:
Handle job output paths and properties
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/lpcraft/+git/lpcraft/+merge/412439
The main complexity here is in ensuring that paths extracted from completed jobs can't escape the build tree, either directly or via symlinks.
I added a `LocalExecuteRun` gadget because we now need to do some work in the container via carefully-crafted `find` and `readlink` subprocesses, and mocking those meant that we weren't really testing the interesting bits. Now we at least do real subprocess execution in those tests, albeit without a container.
`distribute`, `channels`, and `expires` will be implemented on the Launchpad side.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/lpcraft:output into lpcraft:main.
diff --git a/lpcraft/commands/run.py b/lpcraft/commands/run.py
index b1fe013..d8b8a18 100644
--- a/lpcraft/commands/run.py
+++ b/lpcraft/commands/run.py
@@ -1,18 +1,165 @@
# Copyright 2021 Canonical Ltd. This software is licensed under the
# GNU General Public License version 3 (see the file LICENSE).
+import fnmatch
+import io
+import json
+import os
from argparse import Namespace
-from pathlib import Path
+from pathlib import Path, PurePath
+from typing import List, Set
from craft_cli import emit
+from craft_providers import Executor
+from dotenv import dotenv_values
from lpcraft import env
-from lpcraft.config import Config
+from lpcraft.config import Config, Output
from lpcraft.errors import CommandError
from lpcraft.providers import get_provider
from lpcraft.utils import get_host_architecture
+def _check_path_escape(path: PurePath, container: PurePath) -> PurePath:
+ """Check that `path` does not escape `container`.
+
+ Any symlinks in `path` must already have been resolved within the
+ context of the container.
+
+ :raises CommandError: if `path` is outside `container` when fully
+ resolved.
+ :return: A version of `path` relative to `container`.
+ """
+ try:
+ return path.relative_to(container)
+ except ValueError as e:
+ raise CommandError(str(e), retcode=1)
+
+
+def _find_in_instance(instance: Executor, path: Path) -> List[PurePath]:
+ """Find entries in `path` on `instance`.
+
+ :param instance: Provider instance to search.
+ :param path: Path to directory to search.
+ :return: List of non-directory paths found, relative to `path`.
+ """
+ cmd = ["find", str(path), "-mindepth", "1"]
+ # Exclude directories.
+ cmd.extend(["!", "-type", "d"])
+ # Produce unambiguous output: file name relative to the starting path,
+ # terminated by NUL.
+ cmd.extend(["-printf", "%P\\0"])
+ paths = (
+ instance.execute_run(cmd, capture_output=True, check=True)
+ .stdout.rstrip(b"\0")
+ .split(b"\0")
+ )
+ return [PurePath(os.fsdecode(p)) for p in paths]
+
+
+def _resolve_symlinks(
+ instance: Executor, paths: List[PurePath]
+) -> List[PurePath]:
+ """Resolve symlinks in each of `paths` on `instance`.
+
+ Similar to `Path.resolve`, but doesn't require a Python process on
+ `instance`.
+
+ :param instance: Provider instance to inspect.
+ :param paths: Paths to dereference.
+ :return: Dereferenced version of each of the input paths.
+ """
+ paths = (
+ instance.execute_run(
+ ["readlink", "-f", "-z", "--"] + [str(path) for path in paths],
+ capture_output=True,
+ check=True,
+ )
+ .stdout.rstrip(b"\0")
+ .split(b"\0")
+ )
+ return [PurePath(os.fsdecode(p)) for p in paths]
+
+
+def _copy_output_paths(
+ output: Output, remote_cwd: Path, instance: Executor, output_path: Path
+) -> None:
+ """Copy designated output paths from a completed job."""
+ if output.paths is None:
+ return
+
+ for path_pattern in output.paths:
+ # We'll also check individual glob expansions, but checking the
+ # pattern as a whole first produces clearer error messages. We have
+ # to use os.path for this, as pathlib doesn't expose any equivalent
+ # of normpath.
+ _check_path_escape(
+ PurePath(os.path.normpath(remote_cwd / path_pattern)),
+ remote_cwd,
+ )
+
+ remote_paths = sorted(_find_in_instance(instance, remote_cwd))
+ output_files = output_path / "files"
+
+ filtered_paths: Set[PurePath] = set()
+ for path_pattern in output.paths:
+ filtered_paths.update(
+ PurePath(name)
+ for name in fnmatch.filter(
+ [str(path) for path in remote_paths],
+ path_pattern,
+ )
+ )
+ resolved_paths = _resolve_symlinks(
+ instance,
+ [remote_cwd / path for path in sorted(filtered_paths)],
+ )
+
+ for path in sorted(resolved_paths):
+ relative_path = _check_path_escape(path, remote_cwd)
+ destination = output_files / relative_path
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ try:
+ # Path() here works around
+ # https://github.com/canonical/craft-providers/pull/83.
+ instance.pull_file(source=Path(path), destination=destination)
+ except Exception as e:
+ raise CommandError(str(e), retcode=1)
+
+
+def _copy_output_properties(
+ output: Output, remote_cwd: Path, instance: Executor, output_path: Path
+) -> None:
+ """Copy designated output properties from a completed job."""
+ properties = dict(output.properties or {})
+
+ if output.dynamic_properties:
+ [path] = _resolve_symlinks(
+ instance,
+ [remote_cwd / output.dynamic_properties],
+ )
+ _check_path_escape(path, remote_cwd)
+ dynamic_properties = dotenv_values(
+ stream=io.StringIO(
+ instance.execute_run(
+ ["cat", str(path)],
+ capture_output=True,
+ text=True,
+ ).stdout
+ )
+ )
+ properties.update(
+ {
+ key: value
+ for key, value in dynamic_properties.items()
+ if value is not None
+ }
+ )
+
+ with open(output_path / "properties", "w") as f:
+ json.dump(properties, f)
+
+
def run(args: Namespace) -> int:
"""Run a pipeline, launching managed environments as needed."""
config = Config.load(Path(".launchpad.yaml"))
@@ -36,6 +183,7 @@ def run(args: Namespace) -> int:
)
cmd = ["bash", "--noprofile", "--norc", "-ec", job.run]
+ remote_cwd = env.get_managed_environment_project_path()
emit.progress(
f"Launching environment for {job.series}/{host_architecture}"
@@ -50,7 +198,7 @@ def run(args: Namespace) -> int:
with emit.open_stream(f"Running {cmd}") as stream:
proc = instance.execute_run(
cmd,
- cwd=env.get_managed_environment_project_path(),
+ cwd=remote_cwd,
stdout=stream,
stderr=stream,
)
@@ -62,4 +210,16 @@ def run(args: Namespace) -> int:
retcode=proc.returncode,
)
+ if job.output is not None and args.output is not None:
+ output_path = (
+ args.output / job_name / job.series / host_architecture
+ )
+ output_path.mkdir(parents=True, exist_ok=True)
+ _copy_output_paths(
+ job.output, remote_cwd, instance, output_path
+ )
+ _copy_output_properties(
+ job.output, remote_cwd, instance, output_path
+ )
+
return 0
diff --git a/lpcraft/commands/tests/test_run.py b/lpcraft/commands/tests/test_run.py
index 7d19987..3f35b75 100644
--- a/lpcraft/commands/tests/test_run.py
+++ b/lpcraft/commands/tests/test_run.py
@@ -1,16 +1,23 @@
# Copyright 2021 Canonical Ltd. This software is licensed under the
# GNU General Public License version 3 (see the file LICENSE).
+import json
import os
import subprocess
from pathlib import Path
from textwrap import dedent
-from typing import Optional
+from typing import Any, AnyStr, Dict, List, Optional
from unittest.mock import ANY, Mock, call, patch
from craft_providers.lxd import LXC, launch
from fixtures import TempDir
-from testtools.matchers import MatchesStructure
+from testtools.matchers import (
+ AfterPreprocessing,
+ Contains,
+ Equals,
+ MatchesListwise,
+ MatchesStructure,
+)
from lpcraft.commands.tests import CommandBaseTestCase
from lpcraft.errors import CommandError, YAMLError
@@ -18,13 +25,51 @@ from lpcraft.providers._lxd import LXDProvider, _LXDLauncher
from lpcraft.providers.tests import FakeLXDInstaller
+class LocalExecuteRun:
+ """A fake LXDInstance.execute_run that runs subprocesses locally.
+
+ This allows us to set up a temporary directory with the expected
+ contents and then run processes in it more or less normally. Don't run
+ complicated build commands using this, but ordinary system utilities are
+ fine.
+ """
+
+ def __init__(self, override_cwd: Path):
+ super().__init__()
+ self.override_cwd = override_cwd
+ self.call_args_list: List[Any] = []
+
+ def __call__(
+ self,
+ command: List[str],
+ *,
+ cwd: Optional[Path] = None,
+ env: Optional[Dict[str, Optional[str]]] = None,
+ **kwargs: Any
+ ) -> "subprocess.CompletedProcess[AnyStr]":
+ run_kwargs = kwargs.copy()
+ run_kwargs["cwd"] = self.override_cwd
+ if env is not None: # pragma: no cover
+ full_env = os.environ.copy()
+ for key, value in env.items():
+ if value is None:
+ full_env.pop(key, None)
+ else:
+ full_env[key] = value
+ run_kwargs["env"] = full_env
+ self.call_args_list.append(call(command, **run_kwargs))
+ return subprocess.run(command, **run_kwargs)
+
+
class TestRun(CommandBaseTestCase):
def setUp(self):
super().setUp()
- tmp_project_dir = self.useFixture(TempDir()).join("test-project")
- os.mkdir(tmp_project_dir)
- cwd = os.getcwd()
- os.chdir(tmp_project_dir)
+ self.tmp_project_path = Path(
+ self.useFixture(TempDir()).join("test-project")
+ )
+ self.tmp_project_path.mkdir()
+ cwd = Path.cwd()
+ os.chdir(self.tmp_project_path)
self.addCleanup(os.chdir, cwd)
def makeLXDProvider(
@@ -369,3 +414,558 @@ class TestRun(CommandBaseTestCase):
],
execute_run.call_args_list,
)
+
+ @patch("lpcraft.env.get_managed_environment_project_path")
+ @patch("lpcraft.commands.run.get_provider")
+ @patch("lpcraft.commands.run.get_host_architecture", return_value="amd64")
+ def test_copies_output_paths(
+ self,
+ mock_get_host_architecture,
+ mock_get_provider,
+ mock_get_project_path,
+ ):
+ def fake_pull_file(source: Path, destination: Path) -> None:
+ destination.touch()
+
+ output_path = Path(self.useFixture(TempDir()).path)
+ launcher = Mock(spec=launch)
+ provider = self.makeLXDProvider(lxd_launcher=launcher)
+ mock_get_provider.return_value = provider
+ execute_run = LocalExecuteRun(self.tmp_project_path)
+ launcher.return_value.execute_run = execute_run
+ mock_get_project_path.return_value = self.tmp_project_path
+ launcher.return_value.pull_file.side_effect = fake_pull_file
+ config = dedent(
+ """
+ pipeline:
+ - build
+
+ jobs:
+ build:
+ series: focal
+ architectures: amd64
+ run: |
+ true
+ output:
+ paths: ["*.tar.gz", "*.whl"]
+ """
+ )
+ Path(".launchpad.yaml").write_text(config)
+ Path("test_1.0.tar.gz").write_bytes(b"")
+ Path("test_1.0.whl").write_bytes(b"")
+
+ result = self.run_command("run", "--output", str(output_path))
+
+ self.assertEqual(0, result.exit_code)
+ self.assertEqual(
+ [
+ call(
+ ["bash", "--noprofile", "--norc", "-ec", "true\n"],
+ cwd=self.tmp_project_path,
+ stdout=ANY,
+ stderr=ANY,
+ ),
+ call(
+ [
+ "find",
+ str(self.tmp_project_path),
+ "-mindepth",
+ "1",
+ "!",
+ "-type",
+ "d",
+ "-printf",
+ "%P\\0",
+ ],
+ cwd=ANY,
+ capture_output=True,
+ check=True,
+ ),
+ call(
+ [
+ "readlink",
+ "-f",
+ "-z",
+ "--",
+ str(self.tmp_project_path / "test_1.0.tar.gz"),
+ str(self.tmp_project_path / "test_1.0.whl"),
+ ],
+ cwd=ANY,
+ capture_output=True,
+ check=True,
+ ),
+ ],
+ execute_run.call_args_list,
+ )
+ local_output = output_path / "build" / "focal" / "amd64"
+ self.assertEqual(
+ [
+ call(
+ source=self.tmp_project_path / "test_1.0.tar.gz",
+ destination=local_output / "files" / "test_1.0.tar.gz",
+ ),
+ call(
+ source=self.tmp_project_path / "test_1.0.whl",
+ destination=local_output / "files" / "test_1.0.whl",
+ ),
+ ],
+ launcher.return_value.pull_file.call_args_list,
+ )
+ self.assertEqual(
+ ["files", "properties"],
+ sorted(path.name for path in local_output.iterdir()),
+ )
+ self.assertEqual(
+ ["test_1.0.tar.gz", "test_1.0.whl"],
+ sorted(path.name for path in (local_output / "files").iterdir()),
+ )
+
+ @patch("lpcraft.env.get_managed_environment_project_path")
+ @patch("lpcraft.commands.run.get_provider")
+ @patch("lpcraft.commands.run.get_host_architecture", return_value="amd64")
+ def test_output_path_escapes_directly(
+ self,
+ mock_get_host_architecture,
+ mock_get_provider,
+ mock_get_project_path,
+ ):
+ output_path = Path(self.useFixture(TempDir()).path)
+ launcher = Mock(spec=launch)
+ provider = self.makeLXDProvider(lxd_launcher=launcher)
+ mock_get_provider.return_value = provider
+ execute_run = LocalExecuteRun(self.tmp_project_path)
+ launcher.return_value.execute_run = execute_run
+ mock_get_project_path.return_value = self.tmp_project_path
+ config = dedent(
+ """
+ pipeline:
+ - build
+
+ jobs:
+ build:
+ series: focal
+ architectures: amd64
+ run: |
+ true
+ output:
+ paths: ["../../etc/shadow"]
+ """
+ )
+ Path(".launchpad.yaml").write_text(config)
+
+ result = self.run_command("run", "--output", str(output_path))
+
+ # The exact error message differs between Python 3.8 and 3.9, so
+ # don't test it in detail, but make sure it includes the offending
+ # path.
+ self.assertThat(
+ result,
+ MatchesStructure(
+ exit_code=Equals(1),
+ errors=MatchesListwise(
+ [AfterPreprocessing(str, Contains("/etc/shadow"))]
+ ),
+ ),
+ )
+
+ @patch("lpcraft.env.get_managed_environment_project_path")
+ @patch("lpcraft.commands.run.get_provider")
+ @patch("lpcraft.commands.run.get_host_architecture", return_value="amd64")
+ def test_output_path_escapes_symlink(
+ self,
+ mock_get_host_architecture,
+ mock_get_provider,
+ mock_get_project_path,
+ ):
+ output_path = Path(self.useFixture(TempDir()).path)
+ launcher = Mock(spec=launch)
+ provider = self.makeLXDProvider(lxd_launcher=launcher)
+ mock_get_provider.return_value = provider
+ execute_run = LocalExecuteRun(self.tmp_project_path)
+ launcher.return_value.execute_run = execute_run
+ mock_get_project_path.return_value = self.tmp_project_path
+ config = dedent(
+ """
+ pipeline:
+ - build
+
+ jobs:
+ build:
+ series: focal
+ architectures: amd64
+ run: |
+ true
+ output:
+ paths: ["*.txt"]
+ """
+ )
+ Path(".launchpad.yaml").write_text(config)
+ Path("symlink.txt").symlink_to("../target.txt")
+
+ result = self.run_command("run", "--output", str(output_path))
+
+ # The exact error message differs between Python 3.8 and 3.9, so
+ # don't test it in detail, but make sure it includes the offending
+ # path.
+ self.assertThat(
+ result,
+ MatchesStructure(
+ exit_code=Equals(1),
+ errors=MatchesListwise(
+ [AfterPreprocessing(str, Contains("/target.txt"))]
+ ),
+ ),
+ )
+ self.assertEqual(
+ [
+ call(
+ ["bash", "--noprofile", "--norc", "-ec", "true\n"],
+ cwd=self.tmp_project_path,
+ stdout=ANY,
+ stderr=ANY,
+ ),
+ call(
+ [
+ "find",
+ str(self.tmp_project_path),
+ "-mindepth",
+ "1",
+ "!",
+ "-type",
+ "d",
+ "-printf",
+ "%P\\0",
+ ],
+ cwd=ANY,
+ capture_output=True,
+ check=True,
+ ),
+ call(
+ [
+ "readlink",
+ "-f",
+ "-z",
+ "--",
+ str(self.tmp_project_path / "symlink.txt"),
+ ],
+ cwd=ANY,
+ capture_output=True,
+ check=True,
+ ),
+ ],
+ execute_run.call_args_list,
+ )
+
+ @patch("lpcraft.env.get_managed_environment_project_path")
+ @patch("lpcraft.commands.run.get_provider")
+ @patch("lpcraft.commands.run.get_host_architecture", return_value="amd64")
+ def test_output_path_pull_file_fails(
+ self,
+ mock_get_host_architecture,
+ mock_get_provider,
+ mock_get_project_path,
+ ):
+ output_path = Path(self.useFixture(TempDir()).path)
+ launcher = Mock(spec=launch)
+ provider = self.makeLXDProvider(lxd_launcher=launcher)
+ mock_get_provider.return_value = provider
+ execute_run = LocalExecuteRun(self.tmp_project_path)
+ launcher.return_value.execute_run = execute_run
+ launcher.return_value.pull_file.side_effect = FileNotFoundError(
+ "File not found"
+ )
+ mock_get_project_path.return_value = self.tmp_project_path
+ config = dedent(
+ """
+ pipeline:
+ - build
+
+ jobs:
+ build:
+ series: focal
+ architectures: amd64
+ run: |
+ true
+ output:
+ paths: ["*.whl"]
+ """
+ )
+ Path(".launchpad.yaml").write_text(config)
+ Path("test_1.0.whl").write_bytes(b"")
+
+ result = self.run_command("run", "--output", str(output_path))
+
+ self.assertThat(
+ result,
+ MatchesStructure.byEquality(
+ exit_code=1, errors=[CommandError("File not found", retcode=1)]
+ ),
+ )
+
+ @patch("lpcraft.env.get_managed_environment_project_path")
+ @patch("lpcraft.commands.run.get_provider")
+ @patch("lpcraft.commands.run.get_host_architecture", return_value="amd64")
+ def test_reads_properties(
+ self,
+ mock_get_host_architecture,
+ mock_get_provider,
+ mock_get_project_path,
+ ):
+ output_path = Path(self.useFixture(TempDir()).path)
+ launcher = Mock(spec=launch)
+ provider = self.makeLXDProvider(lxd_launcher=launcher)
+ mock_get_provider.return_value = provider
+ execute_run = LocalExecuteRun(self.tmp_project_path)
+ launcher.return_value.execute_run = execute_run
+ mock_get_project_path.return_value = self.tmp_project_path
+ config = dedent(
+ """
+ pipeline:
+ - build
+
+ jobs:
+ build:
+ series: focal
+ architectures: amd64
+ run: |
+ true
+ output:
+ properties:
+ foo: bar
+ """
+ )
+ Path(".launchpad.yaml").write_text(config)
+
+ result = self.run_command("run", "--output", str(output_path))
+
+ self.assertEqual(0, result.exit_code)
+ self.assertEqual(
+ [
+ call(
+ ["bash", "--noprofile", "--norc", "-ec", "true\n"],
+ cwd=self.tmp_project_path,
+ stdout=ANY,
+ stderr=ANY,
+ ),
+ ],
+ execute_run.call_args_list,
+ )
+ local_output = output_path / "build" / "focal" / "amd64"
+ self.assertEqual(
+ {"foo": "bar"},
+ json.loads((local_output / "properties").read_text()),
+ )
+
+ @patch("lpcraft.env.get_managed_environment_project_path")
+ @patch("lpcraft.commands.run.get_provider")
+ @patch("lpcraft.commands.run.get_host_architecture", return_value="amd64")
+ def test_reads_dynamic_properties(
+ self,
+ mock_get_host_architecture,
+ mock_get_provider,
+ mock_get_project_path,
+ ):
+ output_path = Path(self.useFixture(TempDir()).path)
+ launcher = Mock(spec=launch)
+ provider = self.makeLXDProvider(lxd_launcher=launcher)
+ mock_get_provider.return_value = provider
+ execute_run = LocalExecuteRun(self.tmp_project_path)
+ launcher.return_value.execute_run = execute_run
+ mock_get_project_path.return_value = self.tmp_project_path
+ config = dedent(
+ """
+ pipeline:
+ - test
+
+ jobs:
+ test:
+ series: focal
+ architectures: amd64
+ run: |
+ true
+ output:
+ dynamic-properties: properties
+ """
+ )
+ Path(".launchpad.yaml").write_text(config)
+ Path("properties").write_text("version=0.1\n")
+
+ result = self.run_command("run", "--output", str(output_path))
+
+ self.assertEqual(0, result.exit_code)
+ self.assertEqual(
+ [
+ call(
+ ["bash", "--noprofile", "--norc", "-ec", "true\n"],
+ cwd=self.tmp_project_path,
+ stdout=ANY,
+ stderr=ANY,
+ ),
+ call(
+ [
+ "readlink",
+ "-f",
+ "-z",
+ "--",
+ str(self.tmp_project_path / "properties"),
+ ],
+ cwd=ANY,
+ capture_output=True,
+ check=True,
+ ),
+ call(
+ ["cat", str(self.tmp_project_path / "properties")],
+ cwd=ANY,
+ capture_output=True,
+ text=True,
+ ),
+ ],
+ execute_run.call_args_list,
+ )
+ local_output = output_path / "test" / "focal" / "amd64"
+ self.assertEqual(
+ {"version": "0.1"},
+ json.loads((local_output / "properties").read_text()),
+ )
+
+ @patch("lpcraft.env.get_managed_environment_project_path")
+ @patch("lpcraft.commands.run.get_provider")
+ @patch("lpcraft.commands.run.get_host_architecture", return_value="amd64")
+ def test_dynamic_properties_override_properties(
+ self,
+ mock_get_host_architecture,
+ mock_get_provider,
+ mock_get_project_path,
+ ):
+ output_path = Path(self.useFixture(TempDir()).path)
+ launcher = Mock(spec=launch)
+ provider = self.makeLXDProvider(lxd_launcher=launcher)
+ mock_get_provider.return_value = provider
+ execute_run = LocalExecuteRun(self.tmp_project_path)
+ launcher.return_value.execute_run = execute_run
+ mock_get_project_path.return_value = self.tmp_project_path
+ config = dedent(
+ """
+ pipeline:
+ - test
+
+ jobs:
+ test:
+ series: focal
+ architectures: amd64
+ run: |
+ true
+ output:
+ properties:
+ version: "0.1"
+ dynamic-properties: properties
+ """
+ )
+ Path(".launchpad.yaml").write_text(config)
+ Path("properties").write_text("version=0.2\n")
+
+ result = self.run_command("run", "--output", str(output_path))
+
+ self.assertEqual(0, result.exit_code)
+ local_output = output_path / "test" / "focal" / "amd64"
+ self.assertEqual(
+ {"version": "0.2"},
+ json.loads((local_output / "properties").read_text()),
+ )
+
+ @patch("lpcraft.env.get_managed_environment_project_path")
+ @patch("lpcraft.commands.run.get_provider")
+ @patch("lpcraft.commands.run.get_host_architecture", return_value="amd64")
+ def test_run_dynamic_properties_escapes_directly(
+ self,
+ mock_get_host_architecture,
+ mock_get_provider,
+ mock_get_project_path,
+ ):
+ output_path = Path(self.useFixture(TempDir()).path)
+ launcher = Mock(spec=launch)
+ provider = self.makeLXDProvider(lxd_launcher=launcher)
+ mock_get_provider.return_value = provider
+ execute_run = LocalExecuteRun(self.tmp_project_path)
+ launcher.return_value.execute_run = execute_run
+ mock_get_project_path.return_value = self.tmp_project_path
+ config = dedent(
+ """
+ pipeline:
+ - test
+
+ jobs:
+ test:
+ series: focal
+ architectures: amd64
+ run: |
+ true
+ output:
+ dynamic-properties: ../properties
+ """
+ )
+ Path(".launchpad.yaml").write_text(config)
+
+ result = self.run_command("run", "--output", str(output_path))
+
+ # The exact error message differs between Python 3.8 and 3.9, so
+ # don't test it in detail, but make sure it includes the offending
+ # path.
+ self.assertThat(
+ result,
+ MatchesStructure(
+ exit_code=Equals(1),
+ errors=MatchesListwise(
+ [AfterPreprocessing(str, Contains("/properties"))]
+ ),
+ ),
+ )
+
+ @patch("lpcraft.env.get_managed_environment_project_path")
+ @patch("lpcraft.commands.run.get_provider")
+ @patch("lpcraft.commands.run.get_host_architecture", return_value="amd64")
+ def test_run_dynamic_properties_escapes_symlink(
+ self,
+ mock_get_host_architecture,
+ mock_get_provider,
+ mock_get_project_path,
+ ):
+ output_path = Path(self.useFixture(TempDir()).path)
+ launcher = Mock(spec=launch)
+ provider = self.makeLXDProvider(lxd_launcher=launcher)
+ mock_get_provider.return_value = provider
+ execute_run = LocalExecuteRun(self.tmp_project_path)
+ launcher.return_value.execute_run = execute_run
+ mock_get_project_path.return_value = self.tmp_project_path
+ config = dedent(
+ """
+ pipeline:
+ - test
+
+ jobs:
+ test:
+ series: focal
+ architectures: amd64
+ run: |
+ true
+ output:
+ dynamic-properties: properties
+ """
+ )
+ Path(".launchpad.yaml").write_text(config)
+ Path("properties").symlink_to("../target")
+
+ result = self.run_command("run", "--output", str(output_path))
+
+ # The exact error message differs between Python 3.8 and 3.9, so
+ # don't test it in detail, but make sure it includes the offending
+ # path.
+ self.assertThat(
+ result,
+ MatchesStructure(
+ exit_code=Equals(1),
+ errors=MatchesListwise(
+ [AfterPreprocessing(str, Contains("/target"))]
+ ),
+ ),
+ )
diff --git a/lpcraft/config.py b/lpcraft/config.py
index 0a6d0af..780398f 100644
--- a/lpcraft/config.py
+++ b/lpcraft/config.py
@@ -1,6 +1,8 @@
# Copyright 2021 Canonical Ltd. This software is licensed under the
# GNU General Public License version 3 (see the file LICENSE).
+from datetime import timedelta
+from enum import Enum
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
@@ -20,12 +22,36 @@ class ModelConfigDefaults(
"""Define lpcraft's model defaults."""
+class OutputDistributeEnum(Enum):
+ """Valid values for `output.distribute.`"""
+
+ artifactory = "artifactory"
+
+
+class Output(ModelConfigDefaults):
+ """Job output properties."""
+
+ paths: Optional[List[StrictStr]]
+ distribute: Optional[OutputDistributeEnum]
+ channels: Optional[List[StrictStr]]
+ properties: Optional[Dict[StrictStr, StrictStr]]
+ dynamic_properties: Optional[Path]
+ expires: Optional[timedelta]
+
+ @pydantic.validator("expires")
+ def validate_expires(cls, v: timedelta) -> timedelta:
+ if v < timedelta(0):
+ raise ValueError("non-negative duration expected")
+ return v
+
+
class Job(ModelConfigDefaults):
"""A job definition."""
series: StrictStr
architectures: List[StrictStr]
run: Optional[StrictStr]
+ output: Optional[Output]
@pydantic.validator("architectures", pre=True)
def validate_architectures(
diff --git a/lpcraft/main.py b/lpcraft/main.py
index cf6dcf8..124be7e 100644
--- a/lpcraft/main.py
+++ b/lpcraft/main.py
@@ -5,6 +5,7 @@
import logging
from argparse import ArgumentParser
+from pathlib import Path
from craft_cli import CraftError, EmitterMode, emit
@@ -61,6 +62,10 @@ def main() -> int:
# alongside the individual subcommands rather than here.
parser_run = subparsers.add_parser("run", help=run.__doc__)
+ parser_run.add_argument(
+ "--output", type=Path, help="Write output files to this directory."
+ )
+
parser_run.set_defaults(func=run)
parser_version = subparsers.add_parser("version", help=version.__doc__)
diff --git a/lpcraft/tests/test_config.py b/lpcraft/tests/test_config.py
index 6fb98f9..f36ce69 100644
--- a/lpcraft/tests/test_config.py
+++ b/lpcraft/tests/test_config.py
@@ -1,10 +1,12 @@
# Copyright 2021 Canonical Ltd. This software is licensed under the
# GNU General Public License version 3 (see the file LICENSE).
+from datetime import timedelta
from pathlib import Path
from textwrap import dedent
from fixtures import TempDir
+from pydantic import ValidationError
from testtools import TestCase
from testtools.matchers import (
Equals,
@@ -13,7 +15,7 @@ from testtools.matchers import (
MatchesStructure,
)
-from lpcraft.config import Config
+from lpcraft.config import Config, OutputDistributeEnum
class TestConfig(TestCase):
@@ -127,3 +129,63 @@ class TestConfig(TestCase):
),
),
)
+
+ def test_output(self):
+ path = self.create_config(
+ dedent(
+ """
+ pipeline:
+ - build
+
+ jobs:
+ build:
+ series: focal
+ architectures: [amd64]
+ run: pyproject-build
+ output:
+ paths: ["*.whl"]
+ distribute: artifactory
+ channels: [edge]
+ properties:
+ foo: bar
+ dynamic-properties: properties
+ expires: 1:00:00
+ """
+ )
+ )
+ config = Config.load(path)
+ self.assertThat(
+ config.jobs["build"][0].output,
+ MatchesStructure.byEquality(
+ paths=["*.whl"],
+ distribute=OutputDistributeEnum.artifactory,
+ channels=["edge"],
+ properties={"foo": "bar"},
+ dynamic_properties=Path("properties"),
+ expires=timedelta(hours=1),
+ ),
+ )
+
+ def test_output_negative_expires(self):
+ path = self.create_config(
+ dedent(
+ """
+ pipeline:
+ - build
+
+ jobs:
+ build:
+ series: focal
+ architectures: [amd64]
+ run: pyproject-build
+ output:
+ expires: -1:00:00
+ """
+ )
+ )
+ self.assertRaisesRegex(
+ ValidationError,
+ r"non-negative duration expected",
+ Config.load,
+ path,
+ )
diff --git a/requirements.in b/requirements.in
index 69dca66..a5db10c 100644
--- a/requirements.in
+++ b/requirements.in
@@ -1,4 +1,5 @@
craft-providers
pydantic
PyYAML
+python-dotenv
git+git://github.com/canonical/craft-cli.git@c172fa00f61dff8e510116d1c23258e79f710e38
diff --git a/requirements.txt b/requirements.txt
index 21ecf3d..107582d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -21,6 +21,8 @@ pydantic==1.8.2
# -r requirements.in
# craft-cli
# craft-providers
+python-dotenv==0.19.2
+ # via -r requirements.in
pyyaml==6.0
# via
# -r requirements.in
diff --git a/setup.cfg b/setup.cfg
index 621e8f3..4d14610 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -26,6 +26,7 @@ install_requires =
craft-cli
craft-providers
pydantic
+ python-dotenv
python_requires = >=3.8
[options.entry_points]
