launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #28500
[Merge] ~jugmac00/lpcraft:add-pre-post-run-hooks into lpcraft:main
Jürgen Gmach has proposed merging ~jugmac00/lpcraft:add-pre-post-run-hooks into lpcraft:main.
Commit message:
This branch adds pre- and post- run hooks to jobs in lpcraft via the `run-before` and `run-after` configuration keys and the `lpcraft_execute_before_run` and `lpcraft_execute_after_run` hooks for plugins to configure.
It also includes a slight refactor of the `run_job` logic to avoid code duplication.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~jugmac00/lpcraft/+git/lpcraft/+merge/423399
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~jugmac00/lpcraft:add-pre-post-run-hooks into lpcraft:main.
diff --git a/NEWS.rst b/NEWS.rst
index 4e89f28..bcec6ce 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -5,7 +5,10 @@ Version history
0.0.15 (unreleased)
===================
-- Nothing yet.
+- Allow ``run-before`` and ``run-after`` in ``.launchpad.yaml`` config.
+
+- Add ``lpcraft_execute_before_run`` and ``lpcraft_execute_after_run`` hooks.
+
0.0.14 (2022-05-18)
===================
diff --git a/docs/configuration.rst b/docs/configuration.rst
index 5d16337..5e02c2b 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -60,10 +60,18 @@ Job definitions
``plugin`` (optional)
A plugin which will be used for this job. See :doc:`../plugins`
+``run-before`` (optional)
+ A string (possibly multi-line) containing shell commands to run for this
+ job prior to the main ``run`` section.
+
``run`` (optional)
A string (possibly multi-line) containing shell commands to run for this
job.
+``run-after`` (optional)
+ A string (possibly multi-line) containing shell commands to run for this
+ job after the main ``run`` section.
+
``output`` (optional)
See the :ref:`output-properties` section below.
diff --git a/lpcraft/commands/run.py b/lpcraft/commands/run.py
index 106ac2f..961b198 100644
--- a/lpcraft/commands/run.py
+++ b/lpcraft/commands/run.py
@@ -9,12 +9,13 @@ import os
import shlex
from argparse import Namespace
from pathlib import Path, PurePath
-from typing import List, Optional, Set
+from typing import Dict, List, Optional, Set
from craft_cli import EmitterMode, emit
-from craft_providers import Executor
+from craft_providers import Executor, lxd
from craft_providers.actions.snap_installer import install_from_store
from dotenv import dotenv_values
+from pluggy import PluginManager
from lpcraft import env
from lpcraft.config import Config, Job, Output
@@ -193,6 +194,108 @@ def _copy_output_properties(
json.dump(properties, f)
+def _resolve_runtime_value(
+ pm: PluginManager, job: Job, hook_name: str, job_property: str
+) -> Optional[str]:
+ command_from_config: Optional[str] = getattr(job, job_property, None)
+ if command_from_config is not None:
+ return command_from_config
+ rv: List[str] = getattr(pm.hook, hook_name)()
+ return next(iter(rv), None)
+
+
+def _install_apt_packages(
+ job_name: str,
+ job: Job,
+ packages: List[str],
+ instance: lxd.LXDInstance,
+ host_architecture: str,
+ remote_cwd: Path,
+ apt_replacement_repositories: Optional[List[str]],
+ environment: Optional[Dict[str, Optional[str]]],
+) -> None:
+ if apt_replacement_repositories:
+ # replace sources.list
+ lines = "\n".join(apt_replacement_repositories) + "\n"
+ with emit.open_stream("Replacing /etc/apt/sources.list") as stream:
+ instance.push_file_io(
+ destination=PurePath("/etc/apt/sources.list"),
+ content=io.BytesIO(lines.encode()),
+ file_mode="0644",
+ group="root",
+ user="root",
+ )
+ # update local repository information
+ apt_update = ["apt", "update"]
+ with emit.open_stream(f"Running {apt_update}") as stream:
+ proc = instance.execute_run(
+ apt_update,
+ cwd=remote_cwd,
+ env=environment,
+ stdout=stream,
+ stderr=stream,
+ )
+ if proc.returncode != 0:
+ raise CommandError(
+ f"Job {job_name!r} for "
+ f"{job.series}/{host_architecture} failed with "
+ f"exit status {proc.returncode} "
+ f"while running `{shlex.join(apt_update)}`.",
+ retcode=proc.returncode,
+ )
+ packages_cmd = ["apt", "install", "-y"] + packages
+ emit.progress("Installing system packages")
+ with emit.open_stream(f"Running {packages_cmd}") as stream:
+ proc = instance.execute_run(
+ packages_cmd,
+ cwd=remote_cwd,
+ env=environment,
+ stdout=stream,
+ stderr=stream,
+ )
+ if proc.returncode != 0:
+ raise CommandError(
+ f"Job {job_name!r} for "
+ f"{job.series}/{host_architecture} failed with "
+ f"exit status {proc.returncode} "
+ f"while running `{shlex.join(packages_cmd)}`.",
+ retcode=proc.returncode,
+ )
+
+
+def _run_instance_command(
+ command: str,
+ job_name: str,
+ job: Job,
+ instance: lxd.LXDInstance,
+ host_architecture: str,
+ remote_cwd: Path,
+ environment: Optional[Dict[str, Optional[str]]],
+) -> None:
+ full_run_cmd = ["bash", "--noprofile", "--norc", "-ec", command]
+ emit.progress("Running command for the job...")
+ original_mode = emit.get_mode()
+ if original_mode == EmitterMode.NORMAL:
+ emit.set_mode(EmitterMode.VERBOSE)
+ with emit.open_stream(f"Running {full_run_cmd}") as stream:
+ proc = instance.execute_run(
+ full_run_cmd,
+ cwd=remote_cwd,
+ env=environment,
+ stdout=stream,
+ stderr=stream,
+ )
+ if original_mode == EmitterMode.NORMAL:
+ emit.set_mode(original_mode)
+ if proc.returncode != 0:
+ raise CommandError(
+ f"Job {job_name!r} for "
+ f"{job.series}/{host_architecture} failed with "
+ f"exit status {proc.returncode}.",
+ retcode=proc.returncode,
+ )
+
+
def _run_job(
job_name: str,
job: Job,
@@ -208,15 +311,24 @@ def _run_job(
if host_architecture not in job.architectures:
return
pm = get_plugin_manager(job)
- # XXX jugmac00 2021-12-17: extract inferring run_command
- run_command = None
-
- run_from_configuration = job.run
- if run_from_configuration is not None:
- run_command = run_from_configuration
- else:
- rv = pm.hook.lpcraft_execute_run()
- run_command = rv and rv[0] or None
+ pre_run_command = _resolve_runtime_value(
+ pm,
+ job,
+ hook_name="lpcraft_execute_before_run",
+ job_property="run_before",
+ )
+ run_command = _resolve_runtime_value(
+ pm,
+ job,
+ hook_name="lpcraft_execute_run",
+ job_property="run",
+ )
+ post_run_command = _resolve_runtime_value(
+ pm,
+ job,
+ hook_name="lpcraft_execute_after_run",
+ job_property="run_after",
+ )
if not run_command:
raise CommandError(
@@ -265,77 +377,27 @@ def _run_job(
)
packages = list(itertools.chain(*pm.hook.lpcraft_install_packages()))
if packages:
- if apt_replacement_repositories:
- # replace sources.list
- lines = "\n".join(apt_replacement_repositories) + "\n"
- with emit.open_stream(
- "Replacing /etc/apt/sources.list"
- ) as stream:
- instance.push_file_io(
- destination=PurePath("/etc/apt/sources.list"),
- content=io.BytesIO(lines.encode()),
- file_mode="0644",
- group="root",
- user="root",
- )
- # update local repository information
- apt_update = ["apt", "update"]
- with emit.open_stream(f"Running {apt_update}") as stream:
- proc = instance.execute_run(
- apt_update,
- cwd=remote_cwd,
- env=environment,
- stdout=stream,
- stderr=stream,
- )
- if proc.returncode != 0:
- raise CommandError(
- f"Job {job_name!r} for "
- f"{job.series}/{host_architecture} failed with "
- f"exit status {proc.returncode} "
- f"while running `{shlex.join(apt_update)}`.",
- retcode=proc.returncode,
- )
- packages_cmd = ["apt", "install", "-y"] + packages
- emit.progress("Installing system packages")
- with emit.open_stream(f"Running {packages_cmd}") as stream:
- proc = instance.execute_run(
- packages_cmd,
- cwd=remote_cwd,
- env=environment,
- stdout=stream,
- stderr=stream,
- )
- if proc.returncode != 0:
- raise CommandError(
- f"Job {job_name!r} for "
- f"{job.series}/{host_architecture} failed with "
- f"exit status {proc.returncode} "
- f"while running `{shlex.join(packages_cmd)}`.",
- retcode=proc.returncode,
- )
- full_run_cmd = ["bash", "--noprofile", "--norc", "-ec", run_command]
- emit.progress("Running the job")
- original_mode = emit.get_mode()
- if original_mode == EmitterMode.NORMAL:
- emit.set_mode(EmitterMode.VERBOSE)
- with emit.open_stream(f"Running {full_run_cmd}") as stream:
- proc = instance.execute_run(
- full_run_cmd,
- cwd=remote_cwd,
- env=environment,
- stdout=stream,
- stderr=stream,
- )
- if original_mode == EmitterMode.NORMAL:
- emit.set_mode(original_mode)
- if proc.returncode != 0:
- raise CommandError(
- f"Job {job_name!r} for "
- f"{job.series}/{host_architecture} failed with "
- f"exit status {proc.returncode}.",
- retcode=proc.returncode,
+ _install_apt_packages(
+ job_name=job_name,
+ job=job,
+ packages=packages,
+ instance=instance,
+ host_architecture=host_architecture,
+ remote_cwd=remote_cwd,
+ apt_replacement_repositories=apt_replacement_repositories,
+ environment=environment,
)
+ for cmd in (pre_run_command, run_command, post_run_command):
+ if cmd:
+ _run_instance_command(
+ command=cmd,
+ job_name=job_name,
+ job=job,
+ instance=instance,
+ host_architecture=host_architecture,
+ remote_cwd=remote_cwd,
+ environment=environment,
+ )
if job.output is not None and output is not None:
target_path = output / job_name / job.series / host_architecture
@@ -401,6 +463,7 @@ def run(args: Namespace) -> int:
emit.error(e)
stage_failed = True
if stage_failed:
+ # FIXME: should we still clean here?
raise CommandError(
f"Some jobs in {stage} failed; stopping.", retcode=1
)
diff --git a/lpcraft/config.py b/lpcraft/config.py
index 07f17ff..06f6e23 100644
--- a/lpcraft/config.py
+++ b/lpcraft/config.py
@@ -67,7 +67,9 @@ class Job(ModelConfigDefaults):
series: _Identifier
architectures: List[_Identifier]
+ run_before: Optional[StrictStr]
run: Optional[StrictStr]
+ run_after: Optional[StrictStr]
environment: Optional[Dict[str, Optional[str]]]
output: Optional[Output]
snaps: Optional[List[StrictStr]]
diff --git a/lpcraft/plugin/hookspecs.py b/lpcraft/plugin/hookspecs.py
index 6b198ea..ddb3b8a 100644
--- a/lpcraft/plugin/hookspecs.py
+++ b/lpcraft/plugin/hookspecs.py
@@ -7,6 +7,8 @@ __all__ = [
"lpcraft_install_packages",
"lpcraft_install_snaps",
"lpcraft_execute_run",
+ "lpcraft_execute_before_run",
+ "lpcraft_execute_after_run",
"lpcraft_set_environment",
]
@@ -41,3 +43,13 @@ def lpcraft_set_environment() -> dict[str, str | None]:
# Please note: when there is the same environment variable provided by
# the plugin and the configuration file, the one in the configuration
# file will be taken into account
+
+
+@hookspec # type: ignore
+def lpcraft_execute_before_run() -> str:
+ """Command to execute prior to the main execution body."""
+
+
+@hookspec # type: ignore
+def lpcraft_execute_after_run() -> str:
+ """Command to execute after the main execution body."""
diff --git a/lpcraft/tests/test_config.py b/lpcraft/tests/test_config.py
index 6867024..d34b718 100644
--- a/lpcraft/tests/test_config.py
+++ b/lpcraft/tests/test_config.py
@@ -59,8 +59,10 @@ class TestConfig(TestCase):
test:
series: focal
architectures: [amd64, arm64]
+ run-before: pip install --upgrade setuptools build
run: |
tox
+ run-after: coverage report
"""
)
)
@@ -76,7 +78,9 @@ class TestConfig(TestCase):
MatchesStructure.byEquality(
series="focal",
architectures=["amd64", "arm64"],
+ run_before="pip install --upgrade setuptools build", # noqa:E501
run="tox\n",
+ run_after="coverage report",
)
]
)
