launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #28589
[Merge] ~jugmac00/lpcraft:add-support-for-conda-channels into lpcraft:main
Jürgen Gmach has proposed merging ~jugmac00/lpcraft:add-support-for-conda-channels into lpcraft:main.
Commit message:
Add support for Conda channels
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~jugmac00/lpcraft/+git/lpcraft/+merge/424380
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~jugmac00/lpcraft:add-support-for-conda-channels into lpcraft:main.
diff --git a/lpcraft/commands/run.py b/lpcraft/commands/run.py
index 4a0ab82..b3a9091 100644
--- a/lpcraft/commands/run.py
+++ b/lpcraft/commands/run.py
@@ -42,6 +42,12 @@ def _check_relative_path(path: PurePath, container: PurePath) -> PurePath:
raise CommandError(str(e), retcode=1)
+def _convert_config_list_to_dict(list_: List[str]) -> Dict[str, str]:
+ # takes a list of strings, each string separated by an equal sign,
+ # and converts it to a dictionary
+ return dict(pair.split("=", maxsplit=1) for pair in list_)
+
+
def _remove_prefix_if_possible(path: PurePath, prefix: str) -> PurePath:
"""Remove an initial prefix from `path` if possible.
@@ -311,6 +317,7 @@ def _run_job(
output: Optional[Path],
apt_replacement_repositories: Optional[List[str]] = None,
env_from_cli: Optional[List[str]] = None,
+ plugin_settings: Optional[List[str]] = None,
) -> None:
"""Run a single job."""
# XXX jugmac00 2022-04-27: we should create a configuration object to be
@@ -318,7 +325,12 @@ def _run_job(
host_architecture = get_host_architecture()
if host_architecture not in job.architectures:
return
- pm = get_plugin_manager(job)
+ # verbosity is necessary to please mypy
+ if plugin_settings is not None:
+ plugin_settings_as_dict = _convert_config_list_to_dict(plugin_settings)
+ pm = get_plugin_manager(job, plugin_settings_as_dict)
+ else:
+ pm = get_plugin_manager(job, None)
pre_run_command = _resolve_runtime_value(
pm,
job,
@@ -357,6 +369,7 @@ def _run_job(
env_from_plugin.update(env_from_configuration)
environment = env_from_plugin
if env_from_cli:
+ # XXX jugmac00 2022-05-13: use _convert_config_list_to_dict
pairs_from_cli = dict(
pair.split("=", maxsplit=1) for pair in env_from_cli
)
@@ -461,6 +474,9 @@ def run(args: Namespace) -> int:
args, "apt_replace_repositories", None
),
env_from_cli=getattr(args, "set_env", None),
+ plugin_settings=getattr(
+ args, "plugin_setting", None
+ ),
)
except CommandError as e:
if len(stage) == 1:
@@ -516,6 +532,7 @@ def run_one(args: Namespace) -> int:
args, "apt_replace_repositories", None
),
env_from_cli=getattr(args, "set_env", None),
+ plugin_settings=getattr(args, "plugin_setting", None),
)
finally:
should_clean_environment = getattr(args, "clean", False)
diff --git a/lpcraft/main.py b/lpcraft/main.py
index c9af819..3cef45d 100644
--- a/lpcraft/main.py
+++ b/lpcraft/main.py
@@ -107,6 +107,11 @@ def main(argv: Optional[List[str]] = None) -> int:
action="append",
help="Set an environment variable.",
)
+ parser_run.add_argument(
+ "--plugin-setting",
+ action="append",
+ help="Add additional plugin setting.",
+ )
parser_run.set_defaults(func=run)
parser_run_one = subparsers.add_parser(
@@ -149,6 +154,11 @@ def main(argv: Optional[List[str]] = None) -> int:
action="append",
help="Set an environment variable.",
)
+ parser_run_one.add_argument(
+ "--plugin-setting",
+ action="append",
+ help="Add additional plugin setting.",
+ )
parser_run_one.set_defaults(func=run_one)
parser_version = subparsers.add_parser(
diff --git a/lpcraft/plugin/manager.py b/lpcraft/plugin/manager.py
index f6dfdd3..036da3e 100644
--- a/lpcraft/plugin/manager.py
+++ b/lpcraft/plugin/manager.py
@@ -1,3 +1,5 @@
+from typing import Dict, Optional
+
import pluggy
from lpcraft.config import Job
@@ -6,7 +8,9 @@ from lpcraft.plugin.lib import InternalPlugins
from lpcraft.plugins import PLUGINS
-def get_plugin_manager(job: Job) -> pluggy.PluginManager:
+def get_plugin_manager(
+ job: Job, plugin_settings: Optional[Dict[str, str]] = None
+) -> pluggy.PluginManager:
pm = pluggy.PluginManager(NAME)
pm.add_hookspecs(hookspecs)
@@ -15,6 +19,6 @@ def get_plugin_manager(job: Job) -> pluggy.PluginManager:
# register builtin plugins
if job.plugin:
- pm.register(PLUGINS[job.plugin](job))
+ pm.register(PLUGINS[job.plugin](job, plugin_settings))
return pm
diff --git a/lpcraft/plugin/tests/test_plugins.py b/lpcraft/plugin/tests/test_plugins.py
index e0ec098..5b28fd7 100644
--- a/lpcraft/plugin/tests/test_plugins.py
+++ b/lpcraft/plugin/tests/test_plugins.py
@@ -528,7 +528,7 @@ class TestPlugins(CommandBaseTestCase):
fi
export PATH=$HOME/miniconda3/bin:$PATH
conda remove --all -q -y -n $CONDA_ENV
- conda create -n $CONDA_ENV -q -y -c conda-forge -c defaults PYTHON=3.8 conda-build mamba pip
+ conda create -n $CONDA_ENV -q -y -c conda-forge -c defaults -c https://user:pass@xxxxxxxxxxxxxxxxxxxxx/artifactory/soss-conda-stable-local/ PYTHON=3.8 conda-build mamba pip
source activate $CONDA_ENV
""" # noqa:E501
)
@@ -536,7 +536,7 @@ class TestPlugins(CommandBaseTestCase):
"""
export PATH=$HOME/miniconda3/bin:$PATH
source activate $CONDA_ENV
- conda-build --no-anaconda-upload --output-folder dist -c conda-forge -c defaults info/recipe/parent
+ conda-build --no-anaconda-upload --output-folder dist -c conda-forge -c defaults -c https://user:pass@xxxxxxxxxxxxxxxxxxxxx/artifactory/soss-conda-stable-local/ info/recipe/parent
pip install --upgrade pytest
""" # noqa: E501
)
@@ -545,88 +545,102 @@ class TestPlugins(CommandBaseTestCase):
"source activate $CONDA_ENV; conda env export"
)
- self.run_command("run")
+ self.run_command(
+ "run",
+ "--plugin-setting",
+ "miniconda_conda_channel=https://user:pass@xxxxxxxxxxxxxxxxxxxxx/artifactory/soss-conda-stable-local/";, # noqa: E501
+ )
self.assertEqual(
- [
- call(
- ["apt", "update"],
- cwd=PosixPath("/root/lpcraft/project"),
- env={"CONDA_ENV": "lpci"},
- stdout=ANY,
- stderr=ANY,
- ),
- call(
- [
- "apt",
- "install",
- "-y",
- "git",
- "python3-dev",
- "python3-pip",
- "python3-venv",
- "wget",
- "automake",
- "build-essential",
- "cmake",
- "gcc",
- "g++",
- "libc++-dev",
- "libc6-dev",
- "libffi-dev",
- "libjpeg-dev",
- "libpng-dev",
- "libreadline-dev",
- "libsqlite3-dev",
- "libtool",
- "zlib1g-dev",
- ],
- cwd=PosixPath("/root/lpcraft/project"),
- env={"CONDA_ENV": "lpci"},
- stdout=ANY,
- stderr=ANY,
- ),
- call(
- [
- "bash",
- "--noprofile",
- "--norc",
- "-ec",
- pre_run_command,
- ],
- cwd=PosixPath("/root/lpcraft/project"),
- env={"CONDA_ENV": "lpci"},
- stdout=ANY,
- stderr=ANY,
- ),
- call(
- [
- "bash",
- "--noprofile",
- "--norc",
- "-ec",
- run_command,
- ],
- cwd=PosixPath("/root/lpcraft/project"),
- env={"CONDA_ENV": "lpci"},
- stdout=ANY,
- stderr=ANY,
- ),
- call(
- [
- "bash",
- "--noprofile",
- "--norc",
- "-ec",
- post_run_command,
- ],
- cwd=PosixPath("/root/lpcraft/project"),
- env={"CONDA_ENV": "lpci"},
- stdout=ANY,
- stderr=ANY,
- ),
- ],
- execute_run.call_args_list,
+ call(
+ ["apt", "update"],
+ cwd=PosixPath("/root/lpcraft/project"),
+ env={"CONDA_ENV": "lpci"},
+ stdout=ANY,
+ stderr=ANY,
+ ),
+ execute_run.call_args_list[0],
+ )
+ self.assertEqual(
+ call(
+ [
+ "apt",
+ "install",
+ "-y",
+ "git",
+ "python3-dev",
+ "python3-pip",
+ "python3-venv",
+ "wget",
+ "automake",
+ "build-essential",
+ "cmake",
+ "gcc",
+ "g++",
+ "libc++-dev",
+ "libc6-dev",
+ "libffi-dev",
+ "libjpeg-dev",
+ "libpng-dev",
+ "libreadline-dev",
+ "libsqlite3-dev",
+ "libtool",
+ "zlib1g-dev",
+ ],
+ cwd=PosixPath("/root/lpcraft/project"),
+ env={"CONDA_ENV": "lpci"},
+ stdout=ANY,
+ stderr=ANY,
+ ),
+ execute_run.call_args_list[1],
+ )
+ self.assertEqual(
+ call(
+ [
+ "bash",
+ "--noprofile",
+ "--norc",
+ "-ec",
+ pre_run_command,
+ ],
+ cwd=PosixPath("/root/lpcraft/project"),
+ env={"CONDA_ENV": "lpci"},
+ stdout=ANY,
+ stderr=ANY,
+ ),
+ execute_run.call_args_list[2],
+ )
+ self.assertEqual(
+ call(
+ [
+ "bash",
+ "--noprofile",
+ "--norc",
+ "-ec",
+ run_command,
+ ],
+ cwd=PosixPath("/root/lpcraft/project"),
+ env={"CONDA_ENV": "lpci"},
+ stdout=ANY,
+ stderr=ANY,
+ ),
+ execute_run.call_args_list[3],
+ )
+ self.assertEqual(
+ call(
+ [
+ "bash",
+ "--noprofile",
+ "--norc",
+ "-ec",
+ post_run_command,
+ ],
+ cwd=PosixPath("/root/lpcraft/project"),
+ env={"CONDA_ENV": "lpci"},
+ stdout=ANY,
+ stderr=ANY,
+ ),
+ execute_run.call_args_list[4],
)
def test_conda_build_plugin_finds_recipe(self):
@@ -902,3 +916,48 @@ class TestPlugins(CommandBaseTestCase):
get_plugin_manager,
config_obj.jobs["build"][0],
)
+
+ @patch("lpcraft.commands.run.get_provider")
+ @patch("lpcraft.commands.run.get_host_architecture", return_value="amd64")
+ def test_additional_settings(
+ self, mock_get_host_architecture, mock_get_provider
+ ):
+ # XXX jugmac00 2022-06-13
+ # this test covers the case when there are additional plugin settings,
+ # but not soss related
+ # this has not (yet) a real use case, but is necessary for coverage
+ launcher = Mock(spec=launch)
+ provider = makeLXDProvider(lxd_launcher=launcher)
+ mock_get_provider.return_value = provider
+ execute_run = launcher.return_value.execute_run
+ execute_run.return_value = subprocess.CompletedProcess([], 0)
+ config = dedent(
+ """
+ pipeline:
+ - build
+
+ jobs:
+ build:
+ series: focal
+ architectures: amd64
+ plugin: conda-build
+ build-target: info/recipe/parent
+ conda-channels:
+ - conda-forge
+ conda-packages:
+ - mamba
+ - pip
+ conda-python: 3.8
+ run: |
+ pip install --upgrade pytest
+ """
+ )
+ Path(".launchpad.yaml").write_text(config)
+
+ result = self.run_command(
+ "run",
+ "--plugin-setting",
+ "foo=bar",
+ )
+
+ self.assertEqual(0, result.exit_code)
diff --git a/lpcraft/plugins/plugins.py b/lpcraft/plugins/plugins.py
index 0797fba..833c461 100644
--- a/lpcraft/plugins/plugins.py
+++ b/lpcraft/plugins/plugins.py
@@ -12,7 +12,7 @@ __all__ = [
import textwrap
from pathlib import Path
-from typing import TYPE_CHECKING, ClassVar, List, Optional, cast
+from typing import TYPE_CHECKING, ClassVar, Dict, List, Optional, cast
import pydantic
from pydantic import StrictStr
@@ -43,8 +43,11 @@ class BasePlugin:
class Config(BaseConfig):
pass
- def __init__(self, config: Job) -> None:
+ def __init__(
+ self, config: Job, plugin_settings: Optional[Dict[str, str]] = None
+ ) -> None:
self.config = config
+ self.additional_settings = plugin_settings
def get_plugin_config(self) -> BaseConfig:
"""Return the properly typecast plugin configuration."""
@@ -167,6 +170,12 @@ class MiniCondaPlugin(BasePlugin):
conda_channels.extend(plugin_config.conda_channels)
for channel in set(self.DEFAULT_CONDA_CHANNELS) - set(conda_channels):
conda_channels.append(channel)
+ if self.additional_settings:
+ soss_channel = self.additional_settings.get(
+ "miniconda_conda_channel"
+ )
+ if soss_channel is not None:
+ conda_channels.append(soss_channel)
return conda_channels
@hookimpl # type: ignore
Follow ups
