launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #28522
[Merge] ~jugmac00/lpcraft:add-conda-build-plugin into lpcraft:main
Jürgen Gmach has proposed merging ~jugmac00/lpcraft:add-conda-build-plugin into lpcraft:main.
Commit message:
Test existing recipe folder with missing meta.yaml
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~jugmac00/lpcraft/+git/lpcraft/+merge/423728
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~jugmac00/lpcraft:add-conda-build-plugin into lpcraft:main.
diff --git a/lpcraft/plugin/tests/test_plugins.py b/lpcraft/plugin/tests/test_plugins.py
index fe7d930..e0ec098 100644
--- a/lpcraft/plugin/tests/test_plugins.py
+++ b/lpcraft/plugin/tests/test_plugins.py
@@ -483,3 +483,422 @@ class TestPlugins(CommandBaseTestCase):
plugin_match[0].conda_channels,
)
self.assertEqual(["PYTHON=3.8", "pip"], plugin_match[0].conda_packages)
+
+ @patch("lpcraft.commands.run.get_provider")
+ @patch("lpcraft.commands.run.get_host_architecture", return_value="amd64")
+ def test_conda_build_plugin(
+ self, mock_get_host_architecture, mock_get_provider
+ ):
+ launcher = Mock(spec=launch)
+ provider = makeLXDProvider(lxd_launcher=launcher)
+ mock_get_provider.return_value = provider
+ execute_run = launcher.return_value.execute_run
+ execute_run.return_value = subprocess.CompletedProcess([], 0)
+ config = dedent(
+ """
+ pipeline:
+ - build
+
+ jobs:
+ build:
+ series: focal
+ architectures: amd64
+ plugin: conda-build
+ build-target: info/recipe/parent
+ conda-channels:
+ - conda-forge
+ conda-packages:
+ - mamba
+ - pip
+ conda-python: 3.8
+ run: |
+ pip install --upgrade pytest
+ """
+ )
+ Path(".launchpad.yaml").write_text(config)
+ Path("info/recipe/parent").mkdir(parents=True)
+ Path("info/recipe/meta.yaml").touch()
+ Path("info/recipe/parent/meta.yaml").touch()
+ pre_run_command = dedent(
+ """
+ if [ ! -d "$HOME/miniconda3" ]; then
+ wget -O /tmp/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+ chmod +x /tmp/miniconda.sh
+ /tmp/miniconda.sh -b
+ fi
+ export PATH=$HOME/miniconda3/bin:$PATH
+ conda remove --all -q -y -n $CONDA_ENV
+ conda create -n $CONDA_ENV -q -y -c conda-forge -c defaults PYTHON=3.8 conda-build mamba pip
+ source activate $CONDA_ENV
+ """ # noqa:E501
+ )
+ run_command = dedent(
+ """
+ export PATH=$HOME/miniconda3/bin:$PATH
+ source activate $CONDA_ENV
+ conda-build --no-anaconda-upload --output-folder dist -c conda-forge -c defaults info/recipe/parent
+ pip install --upgrade pytest
+ """ # noqa: E501
+ )
+ post_run_command = (
+ "export PATH=$HOME/miniconda3/bin:$PATH; "
+ "source activate $CONDA_ENV; conda env export"
+ )
+
+ self.run_command("run")
+
+ self.assertEqual(
+ [
+ call(
+ ["apt", "update"],
+ cwd=PosixPath("/root/lpcraft/project"),
+ env={"CONDA_ENV": "lpci"},
+ stdout=ANY,
+ stderr=ANY,
+ ),
+ call(
+ [
+ "apt",
+ "install",
+ "-y",
+ "git",
+ "python3-dev",
+ "python3-pip",
+ "python3-venv",
+ "wget",
+ "automake",
+ "build-essential",
+ "cmake",
+ "gcc",
+ "g++",
+ "libc++-dev",
+ "libc6-dev",
+ "libffi-dev",
+ "libjpeg-dev",
+ "libpng-dev",
+ "libreadline-dev",
+ "libsqlite3-dev",
+ "libtool",
+ "zlib1g-dev",
+ ],
+ cwd=PosixPath("/root/lpcraft/project"),
+ env={"CONDA_ENV": "lpci"},
+ stdout=ANY,
+ stderr=ANY,
+ ),
+ call(
+ [
+ "bash",
+ "--noprofile",
+ "--norc",
+ "-ec",
+ pre_run_command,
+ ],
+ cwd=PosixPath("/root/lpcraft/project"),
+ env={"CONDA_ENV": "lpci"},
+ stdout=ANY,
+ stderr=ANY,
+ ),
+ call(
+ [
+ "bash",
+ "--noprofile",
+ "--norc",
+ "-ec",
+ run_command,
+ ],
+ cwd=PosixPath("/root/lpcraft/project"),
+ env={"CONDA_ENV": "lpci"},
+ stdout=ANY,
+ stderr=ANY,
+ ),
+ call(
+ [
+ "bash",
+ "--noprofile",
+ "--norc",
+ "-ec",
+ post_run_command,
+ ],
+ cwd=PosixPath("/root/lpcraft/project"),
+ env={"CONDA_ENV": "lpci"},
+ stdout=ANY,
+ stderr=ANY,
+ ),
+ ],
+ execute_run.call_args_list,
+ )
+
+ def test_conda_build_plugin_finds_recipe(self):
+ config = dedent(
+ """
+ pipeline:
+ - build
+
+ jobs:
+ build:
+ series: focal
+ architectures: amd64
+ plugin: conda-build
+ conda-channels:
+ - conda-forge
+ conda-packages:
+ - mamba
+ - pip
+ conda-python: 3.8
+ run: |
+ pip install --upgrade pytest
+ """
+ )
+ config_path = Path(".launchpad.yaml")
+ config_path.write_text(config)
+ Path("include/fake_subdir").mkdir(parents=True)
+ meta_yaml = Path("info/recipe/meta.yaml")
+ meta_yaml.parent.mkdir(parents=True)
+ meta_yaml.touch()
+ config_obj = lpcraft.config.Config.load(config_path)
+ self.assertEqual(config_obj.jobs["build"][0].plugin, "conda-build")
+ pm = get_plugin_manager(config_obj.jobs["build"][0])
+ plugins = pm.get_plugins()
+ plugin_match = [
+ _ for _ in plugins if _.__class__.__name__ == "CondaBuildPlugin"
+ ]
+ self.assertEqual("info/recipe", plugin_match[0].build_target)
+
+ def test_conda_build_plugin_finds_recipe_with_fake_parent(self):
+ config = dedent(
+ """
+ pipeline:
+ - build
+
+ jobs:
+ build:
+ series: focal
+ architectures: amd64
+ plugin: conda-build
+ conda-channels:
+ - conda-forge
+ conda-packages:
+ - mamba
+ - pip
+ conda-python: 3.8
+ run: |
+ pip install --upgrade pytest
+ """
+ )
+ config_path = Path(".launchpad.yaml")
+ config_path.write_text(config)
+ meta_yaml = Path("info/recipe/meta.yaml")
+ meta_yaml.parent.mkdir(parents=True)
+ parent_path = meta_yaml.parent.joinpath("parent")
+ parent_path.mkdir()
+ parent_path.joinpath("some_file.yaml").touch()
+ meta_yaml.touch()
+ config_obj = lpcraft.config.Config.load(config_path)
+ self.assertEqual(config_obj.jobs["build"][0].plugin, "conda-build")
+ pm = get_plugin_manager(config_obj.jobs["build"][0])
+ plugins = pm.get_plugins()
+ plugin_match = [
+ _ for _ in plugins if _.__class__.__name__ == "CondaBuildPlugin"
+ ]
+ self.assertEqual("info/recipe", plugin_match[0].build_target)
+
+ def test_conda_build_plugin_finds_parent_recipe(self):
+ config = dedent(
+ """
+ pipeline:
+ - build
+
+ jobs:
+ build:
+ series: focal
+ architectures: amd64
+ plugin: conda-build
+ conda-channels:
+ - conda-forge
+ conda-packages:
+ - mamba
+ - pip
+ conda-python: 3.8
+ run: |
+ pip install --upgrade pytest
+ """
+ )
+ config_path = Path(".launchpad.yaml")
+ config_path.write_text(config)
+ Path("include/fake_subdir").mkdir(parents=True)
+ meta_yaml = Path("info/recipe/meta.yaml")
+ parent_meta_yaml = meta_yaml.parent.joinpath("parent/meta.yaml")
+ parent_meta_yaml.parent.mkdir(parents=True)
+ meta_yaml.touch()
+ parent_meta_yaml.touch()
+ config_obj = lpcraft.config.Config.load(config_path)
+ self.assertEqual(config_obj.jobs["build"][0].plugin, "conda-build")
+ pm = get_plugin_manager(config_obj.jobs["build"][0])
+ plugins = pm.get_plugins()
+ plugin_match = [
+ _ for _ in plugins if _.__class__.__name__ == "CondaBuildPlugin"
+ ]
+ self.assertEqual("info/recipe/parent", plugin_match[0].build_target)
+
+ def test_conda_build_plugin_uses_child_vars_with_parent_recipe(self):
+ config = dedent(
+ """
+ pipeline:
+ - build
+
+ jobs:
+ build:
+ series: focal
+ architectures: amd64
+ plugin: conda-build
+ conda-channels:
+ - conda-forge
+ conda-packages:
+ - mamba
+ - pip
+ conda-python: 3.8
+ run: |
+ pip install --upgrade pytest
+ """
+ )
+ run_command = dedent(
+ """
+ export PATH=$HOME/miniconda3/bin:$PATH
+ source activate $CONDA_ENV
+ conda-build --no-anaconda-upload --output-folder dist -c conda-forge -c defaults -m info/recipe/parent/conda_build_config.yaml -m info/recipe/conda_build_config.yaml info/recipe/parent
+ pip install --upgrade pytest
+ """ # noqa: E501
+ )
+ config_path = Path(".launchpad.yaml")
+ config_path.write_text(config)
+ Path("include/fake_subdir").mkdir(parents=True)
+ meta_yaml = Path("info/recipe/meta.yaml")
+ variant_config = meta_yaml.with_name("conda_build_config.yaml")
+ parent_meta_yaml = meta_yaml.parent.joinpath("parent/meta.yaml")
+ parent_variant_config = parent_meta_yaml.with_name(
+ "conda_build_config.yaml"
+ )
+ parent_meta_yaml.parent.mkdir(parents=True)
+ meta_yaml.touch()
+ variant_config.touch()
+ parent_variant_config.touch()
+ parent_meta_yaml.touch()
+ config_obj = lpcraft.config.Config.load(config_path)
+ self.assertEqual(config_obj.jobs["build"][0].plugin, "conda-build")
+ pm = get_plugin_manager(config_obj.jobs["build"][0])
+ plugins = pm.get_plugins()
+ plugin_match = [
+ _ for _ in plugins if _.__class__.__name__ == "CondaBuildPlugin"
+ ]
+ self.assertEqual(
+ [parent_variant_config.as_posix(), variant_config.as_posix()],
+ plugin_match[0].build_configs,
+ )
+ self.assertEqual(run_command, plugin_match[0].lpcraft_execute_run())
+
+ def test_conda_build_plugin_renames_recipe_templates(self):
+ config = dedent(
+ """
+ pipeline:
+ - build
+
+ jobs:
+ build:
+ series: focal
+ architectures: amd64
+ plugin: conda-build
+ conda-channels:
+ - conda-forge
+ conda-packages:
+ - mamba
+ - pip
+ conda-python: 3.8
+ run: |
+ pip install --upgrade pytest
+ """
+ )
+ config_path = Path(".launchpad.yaml")
+ config_path.write_text(config)
+ meta_yaml = Path("info/recipe/meta.yaml")
+ template_meta_yaml = meta_yaml.with_name("meta.yaml.template")
+ meta_yaml.parent.mkdir(parents=True)
+ meta_yaml.touch()
+ template_meta_yaml.touch()
+ config_obj = lpcraft.config.Config.load(config_path)
+ self.assertEqual(config_obj.jobs["build"][0].plugin, "conda-build")
+ pm = get_plugin_manager(config_obj.jobs["build"][0])
+ plugins = pm.get_plugins()
+ plugin_match = [
+ _ for _ in plugins if _.__class__.__name__ == "CondaBuildPlugin"
+ ]
+ self.assertEqual("info/recipe", plugin_match[0].build_target)
+ self.assertFalse(template_meta_yaml.is_file())
+
+ def test_conda_build_plugin_raises_error_if_no_recipe(self):
+ config = dedent(
+ """
+ pipeline:
+ - build
+
+ jobs:
+ build:
+ series: focal
+ architectures: amd64
+ plugin: conda-build
+ conda-channels:
+ - conda-forge
+ conda-packages:
+ - mamba
+ - pip
+ conda-python: 3.8
+ run: |
+ pip install --upgrade pytest
+ """
+ )
+ config_path = Path(".launchpad.yaml")
+ config_path.write_text(config)
+ config_obj = lpcraft.config.Config.load(config_path)
+ self.assertRaisesRegex(
+ RuntimeError,
+ "No build target found",
+ get_plugin_manager,
+ config_obj.jobs["build"][0],
+ )
+
+ def test_conda_build_plugin_raises_error_if_no_recipe_in_recipe_folder(
+ self,
+ ):
+ config = dedent(
+ """
+ pipeline:
+ - build
+
+ jobs:
+ build:
+ series: focal
+ architectures: amd64
+ plugin: conda-build
+ conda-channels:
+ - conda-forge
+ conda-packages:
+ - mamba
+ - pip
+ conda-python: 3.8
+ run: |
+ pip install --upgrade pytest
+ """
+ )
+ config_path = Path(".launchpad.yaml")
+ config_path.write_text(config)
+ Path("include/fake_subdir").mkdir(parents=True)
+ # there is a recipe folder, but no meta.yaml file
+ meta_yaml = Path("info/recipe/")
+ meta_yaml.mkdir(parents=True)
+ config_obj = lpcraft.config.Config.load(config_path)
+ self.assertRaisesRegex(
+ RuntimeError,
+ "No build target found",
+ get_plugin_manager,
+ config_obj.jobs["build"][0],
+ )
diff --git a/lpcraft/plugins/plugins.py b/lpcraft/plugins/plugins.py
index c114030..0797fba 100644
--- a/lpcraft/plugins/plugins.py
+++ b/lpcraft/plugins/plugins.py
@@ -3,9 +3,15 @@
from __future__ import annotations # isort:skip
-__all__ = ["ToxPlugin", "PyProjectBuildPlugin", "MiniCondaPlugin"]
+__all__ = [
+ "ToxPlugin",
+ "PyProjectBuildPlugin",
+ "MiniCondaPlugin",
+ "CondaBuildPlugin",
+]
import textwrap
+from pathlib import Path
from typing import TYPE_CHECKING, ClassVar, List, Optional, cast
import pydantic
@@ -213,3 +219,174 @@ class MiniCondaPlugin(BasePlugin):
"export PATH=$HOME/miniconda3/bin:$PATH; "
f"source activate $CONDA_ENV; conda env export{run}"
)
+
+
+@register(name="conda-build")
+class CondaBuildPlugin(MiniCondaPlugin):
+ """Sets up `miniconda3` and performs a `conda-build` on a package.
+
+ Usage:
+ In `.launchpad.yaml`, create the following structure:
+
+ .. code-block:: yaml
+
+ jobs:
+ myjob:
+ plugin: conda-build
+ build-target: info/recipe/parent
+ conda-channels:
+ - conda-forge
+ - defaults
+ conda-packages:
+ - mamba
+ - numpy=1.17
+ - scipy
+ - pip
+ conda-python: 3.8
+ run: |
+ conda install ....
+ pip install --upgrade pytest
+ python -m build .
+ """
+
+ class Config(MiniCondaPlugin.Config):
+ build_target: Optional[StrictStr]
+ conda_channels: Optional[List[StrictStr]]
+ conda_packages: Optional[List[StrictStr]]
+ conda_python: Optional[StrictStr]
+
+ DEFAULT_CONDA_PACKAGES = ("conda-build",)
+
+ def get_plugin_config(self) -> "CondaBuildPlugin.Config":
+ return cast(CondaBuildPlugin.Config, self.config.plugin_config)
+
+ @staticmethod
+ def _has_recipe(dir_: Path) -> bool:
+ return dir_.joinpath("meta.yaml").is_file()
+
+ @staticmethod
+ def _rename_recipe_template(dir_: Path) -> None:
+ # XXX techalchemy 2022-04-01: conda packages which are already built
+ # and subsequently downloaded from the anaconda repositories retain
+ # the templated recipe, at `meta.yaml.template`, but place the
+ # rendered template at `meta.yaml`. The rendered recipes contain
+ # hardcoded paths for a specific build environment and, for our
+ # purposes, are not reusable. We need to render new ones from the
+ # original templates.
+ template_path = dir_.joinpath("meta.yaml.template")
+ if template_path.is_file():
+ template_path.replace(dir_ / "meta.yaml")
+
+ def find_recipe(self) -> Path:
+ def _find_recipe_dir(path: Path) -> Path:
+ for subpath in path.iterdir():
+ if subpath.is_dir():
+ self._rename_recipe_template(subpath)
+ if subpath.name == "recipe" and self._has_recipe(subpath):
+ return subpath
+ try:
+ return _find_recipe_dir(subpath)
+ except FileNotFoundError:
+ continue
+ raise FileNotFoundError
+
+ return _find_recipe_dir(Path("."))
+
+ def find_build_target(self) -> str:
+ def find_parents(pth: Path) -> Path:
+ for parent in pth.iterdir():
+ if parent.is_dir():
+ self._rename_recipe_template(parent)
+ if parent.name == "parent" and self._has_recipe(parent):
+ return parent
+ raise FileNotFoundError(pth.joinpath("meta.yaml"))
+
+ try:
+ recipe = self.find_recipe()
+ except FileNotFoundError:
+ raise RuntimeError("No build target found")
+ try:
+ # XXX techalchemy 2022-04-01: Some conda packages are built as
+ # part of a parent package build process (e.g. `mkl-include` which
+ # is built by `intel_repack`). If you acquire the child package
+ # and attempt to build it (`mkl-include` in this case) it will
+ # fail; you must build the parent instead if it exists
+ return find_parents(recipe).as_posix()
+ except FileNotFoundError:
+ return recipe.as_posix()
+
+ @property
+ def build_configs(self) -> list[str]:
+ try:
+ recipe = self.find_recipe()
+ except FileNotFoundError:
+ return []
+ configs = sorted(
+ recipe.glob("**/conda_build_config.yaml"), reverse=True
+ )
+ return [_.as_posix() for _ in configs]
+
+ @property
+ def build_target(self) -> str:
+ build_target = self.get_plugin_config().build_target
+ if not build_target:
+ return self.find_build_target()
+ return build_target
+
+ @hookimpl # type: ignore
+ def lpcraft_set_environment(self) -> dict[str, str]:
+ # XXX techalchemy 2022-04-01: mypy is struggling with the super() call
+ rv: dict[str, str] = super().lpcraft_set_environment()
+ return rv
+
+ @hookimpl # type: ignore
+ def lpcraft_execute_before_run(self) -> str:
+ # XXX techalchemy 2022-04-01: mypy is struggling with the super() call
+ rv: str = super().lpcraft_execute_before_run()
+ return rv
+
+ @hookimpl # type: ignore
+ def lpcraft_execute_after_run(self) -> str:
+ # XXX techalchemy 2022-04-01: mypy is struggling with the super() call
+ rv: str = super().lpcraft_execute_after_run()
+ return rv
+
+ @hookimpl # type: ignore
+ def lpcraft_install_packages(self) -> list[str]:
+ # XXX techalchemy 2022-04-01: mypy is struggling with the super() call
+ base_packages: list[str] = super().lpcraft_install_packages()
+ base_packages.extend(
+ [
+ "automake",
+ "build-essential",
+ "cmake",
+ "gcc",
+ "g++",
+ "libc++-dev",
+ "libc6-dev",
+ "libffi-dev",
+ "libjpeg-dev",
+ "libpng-dev",
+ "libreadline-dev",
+ "libsqlite3-dev",
+ "libtool",
+ "zlib1g-dev",
+ ]
+ )
+ return base_packages
+
+ @hookimpl # type: ignore
+ def lpcraft_execute_run(self) -> str:
+ conda_channels = " ".join(f"-c {_}" for _ in self.conda_channels)
+ conda_channels = f" {conda_channels}" if conda_channels else ""
+ configs = " ".join(f"-m {_}" for _ in self.build_configs)
+ configs = f" {configs}" if configs else ""
+ build_command = "conda-build --no-anaconda-upload --output-folder dist"
+ run_command = self.config.run or ""
+ return textwrap.dedent(
+ f"""
+ export PATH=$HOME/miniconda3/bin:$PATH
+ source activate $CONDA_ENV
+ {build_command}{conda_channels}{configs} {self.build_target}
+ {run_command}"""
+ )
Follow ups
