launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #32506
[Merge] ~ruinedyourlife/launchpad:craft-recipe-build-job-publishing into launchpad:master
Quentin Debhi has proposed merging ~ruinedyourlife/launchpad:craft-recipe-build-job-publishing into launchpad:master.
Commit message:
Move publishing code to craftrecipebuildjob.py
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~ruinedyourlife/launchpad/+git/launchpad/+merge/485887
The publishing jobs did not have the sufficient permissions to be created. The code which this was copied from (charms) has 2 separate models: one for building, and one for publishing.
The craft build only had one model implemented, which could not allow creating publishing jobs the way it was done for charm upload jobs. This fixes it by separating the two jobs, using the correct classes with the correct permissions for each.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~ruinedyourlife/launchpad:craft-recipe-build-job-publishing into launchpad:master.
diff --git a/lib/lp/crafts/configure.zcml b/lib/lp/crafts/configure.zcml
index a0fca60..e822ffe 100644
--- a/lib/lp/crafts/configure.zcml
+++ b/lib/lp/crafts/configure.zcml
@@ -93,7 +93,15 @@
factory="lp.crafts.model.craftrecipebuildbehaviour.CraftRecipeBuildBehaviour"
permission="zope.Public" />
- <!-- craft-related jobs -->
+ <!-- CraftRecipeBuildMacaroonIssuer -->
+ <lp:securedutility
+ class="lp.crafts.model.craftrecipebuild.CraftRecipeBuildMacaroonIssuer"
+ provides="lp.services.macaroons.interfaces.IMacaroonIssuer"
+ name="craft-recipe-build">
+ <allow interface="lp.services.macaroons.interfaces.IMacaroonIssuerPublic"/>
+ </lp:securedutility>
+
+ <!-- CraftRecipeJob, aka builds -->
<class class="lp.crafts.model.craftrecipejob.CraftRecipeJob">
<allow interface="lp.crafts.interfaces.craftrecipejob.ICraftRecipeJob" />
</class>
@@ -106,24 +114,21 @@
<allow interface="lp.crafts.interfaces.craftrecipejob.ICraftRecipeJob" />
<allow interface="lp.crafts.interfaces.craftrecipejob.ICraftRecipeRequestBuildsJob" />
</class>
- <webservice:register module="lp.crafts.interfaces.webservice" />
-
- <!-- CraftRecipeBuildMacaroonIssuer -->
- <lp:securedutility
- class="lp.crafts.model.craftrecipebuild.CraftRecipeBuildMacaroonIssuer"
- provides="lp.services.macaroons.interfaces.IMacaroonIssuer"
- name="craft-recipe-build">
- <allow interface="lp.services.macaroons.interfaces.IMacaroonIssuerPublic"/>
- </lp:securedutility>
- <!-- CraftPublishingJob -->
+ <!-- CraftRecipeBuildJob, aka publishing -->
+ <class class="lp.crafts.model.craftrecipebuildjob.CraftRecipeBuildJob">
+ <allow interface="lp.crafts.interfaces.craftrecipebuildjob.ICraftRecipeBuildJob" />
+ </class>
<lp:securedutility
- component="lp.crafts.model.craftrecipejob.CraftPublishingJob"
- provides="lp.crafts.interfaces.craftrecipejob.ICraftPublishingJobSource">
- <allow interface="lp.crafts.interfaces.craftrecipejob.ICraftPublishingJobSource" />
+ component="lp.crafts.model.craftrecipebuildjob.CraftPublishingJob"
+ provides="lp.crafts.interfaces.craftrecipebuildjob.ICraftPublishingJobSource">
+ <allow interface="lp.crafts.interfaces.craftrecipebuildjob.ICraftPublishingJobSource" />
</lp:securedutility>
- <class class="lp.crafts.model.craftrecipejob.CraftPublishingJob">
- <allow interface="lp.crafts.interfaces.craftrecipejob.ICraftRecipeJob" />
- <allow interface="lp.crafts.interfaces.craftrecipejob.ICraftPublishingJob" />
+ <class class="lp.crafts.model.craftrecipebuildjob.CraftPublishingJob">
+ <allow interface="lp.crafts.interfaces.craftrecipebuildjob.ICraftRecipeBuildJob" />
+ <allow interface="lp.crafts.interfaces.craftrecipebuildjob.ICraftPublishingJob" />
</class>
+
+ <webservice:register module="lp.crafts.interfaces.webservice" />
+
</configure>
diff --git a/lib/lp/crafts/interfaces/craftrecipebuildjob.py b/lib/lp/crafts/interfaces/craftrecipebuildjob.py
new file mode 100644
index 0000000..c94956d
--- /dev/null
+++ b/lib/lp/crafts/interfaces/craftrecipebuildjob.py
@@ -0,0 +1,57 @@
+# Copyright 2024 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Interfaces for craft recipe build jobs."""
+
+__all__ = [
+ "ICraftPublishingJob",
+ "ICraftPublishingJobSource",
+ "ICraftRecipeBuildJob",
+]
+
+from lazr.restful.fields import Reference
+from zope.interface import Attribute, Interface
+
+from lp import _
+from lp.crafts.interfaces.craftrecipebuild import ICraftRecipeBuild
+from lp.services.job.interfaces.job import IJob, IJobSource, IRunnableJob
+
+
+class ICraftRecipeBuildJob(Interface):
+ """A job related to a craft recipe build."""
+
+ job = Reference(
+ title=_("The common Job attributes."),
+ schema=IJob,
+ required=True,
+ readonly=True,
+ )
+ build = Reference(
+ title=_("The craft recipe build to use for this job."),
+ schema=ICraftRecipeBuild,
+ required=True,
+ readonly=True,
+ )
+ metadata = Attribute(_("A dict of data about the job."))
+
+
+class ICraftPublishingJob(IRunnableJob):
+ """
+ A job that publishes craft recipe build artifacts to external repositories.
+ """
+
+ error_message = Attribute("The error message if the publishing failed.")
+
+ def create(build):
+ """Create a new CraftPublishingJob."""
+
+
+class ICraftPublishingJobSource(IJobSource):
+ """A source for creating and finding CraftPublishingJobs."""
+
+ def create(build):
+ """
+ Publish artifacts from a craft recipe build to external repositories.
+
+ :param build: The build to publish.
+ """
diff --git a/lib/lp/crafts/interfaces/craftrecipejob.py b/lib/lp/crafts/interfaces/craftrecipejob.py
index bbd3ab0..addc6ee 100644
--- a/lib/lp/crafts/interfaces/craftrecipejob.py
+++ b/lib/lp/crafts/interfaces/craftrecipejob.py
@@ -7,8 +7,6 @@ __all__ = [
"ICraftRecipeJob",
"ICraftRecipeRequestBuildsJob",
"ICraftRecipeRequestBuildsJobSource",
- "ICraftPublishingJob",
- "ICraftPublishingJobSource",
]
from lazr.restful.fields import Reference
@@ -138,27 +136,3 @@ class ICraftRecipeRequestBuildsJobSource(IJobSource):
recipe and ID, or its `job_type` is not
`CraftRecipeJobType.REQUEST_BUILDS`.
"""
-
-
-class ICraftPublishingJob(IRunnableJob):
- """
- A job that publishes craft recipe build artifacts to external repositories.
- """
-
- build_id = Attribute("The ID of the build to publish.")
- build = Attribute("The build to publish.")
- error_message = Attribute("The error message if the publishing failed.")
-
- def create(build):
- """Create a new CraftPublishingJob."""
-
-
-class ICraftPublishingJobSource(IJobSource):
- """A source for creating and finding CraftPublishingJobs."""
-
- def create(build):
- """
- Publish artifacts from a craft recipe build to external repositories.
-
- :param build: The build to publish.
- """
diff --git a/lib/lp/crafts/model/craftrecipebuildjob.py b/lib/lp/crafts/model/craftrecipebuildjob.py
new file mode 100644
index 0000000..0135c1f
--- /dev/null
+++ b/lib/lp/crafts/model/craftrecipebuildjob.py
@@ -0,0 +1,489 @@
+"""Craft recipe build jobs."""
+
+__all__ = [
+ "CraftPublishingJob",
+ "CraftRecipeBuildJob",
+ "CraftRecipeBuildJobType",
+]
+
+import json
+import os
+import subprocess
+import tempfile
+from configparser import NoSectionError
+
+import transaction
+from lazr.delegates import delegate_to
+from lazr.enum import DBEnumeratedType, DBItem
+from storm.databases.postgres import JSON
+from storm.locals import Int, Reference
+from zope.interface import implementer, provider
+
+from lp.app.errors import NotFoundError
+from lp.crafts.interfaces.craftrecipebuildjob import (
+ ICraftPublishingJob,
+ ICraftPublishingJobSource,
+ ICraftRecipeBuildJob,
+)
+from lp.registry.interfaces.distributionsourcepackage import (
+ IDistributionSourcePackage,
+)
+from lp.services.config import config
+from lp.services.database.enumcol import DBEnum
+from lp.services.database.interfaces import IPrimaryStore, IStore
+from lp.services.database.stormbase import StormBase
+from lp.services.job.model.job import EnumeratedSubclass, Job
+from lp.services.job.runner import BaseRunnableJob
+
+
+class CraftRecipeBuildJobType(DBEnumeratedType):
+ """Values that `ICraftRecipeBuildJob.job_type` can take."""
+
+ PUBLISH_ARTIFACTS = DBItem(
+ 0,
+ "Publish artifacts to external repositories",
+ )
+
+
+@implementer(ICraftRecipeBuildJob)
+class CraftRecipeBuildJob(StormBase):
+ """See `ICraftRecipeBuildJob`."""
+
+ __storm_table__ = "CraftRecipeBuildJob"
+
+ job_id = Int(name="job", primary=True, allow_none=False)
+ job = Reference(job_id, "Job.id")
+
+ build_id = Int(name="build", allow_none=False)
+ build = Reference(build_id, "CraftRecipeBuild.id")
+
+ job_type = DBEnum(enum=CraftRecipeBuildJobType, allow_none=False)
+
+ metadata = JSON("json_data", allow_none=False)
+
+ def __init__(self, build, job_type, metadata, **job_args):
+ super().__init__()
+ self.job = Job(**job_args)
+ self.build = build
+ self.job_type = job_type
+ self.metadata = metadata
+
+ def makeDerived(self):
+ return CraftRecipeBuildJobDerived.makeSubclass(self)
+
+
+@delegate_to(ICraftRecipeBuildJob)
+class CraftRecipeBuildJobDerived(
+ BaseRunnableJob, metaclass=EnumeratedSubclass
+):
+ """See `ICraftRecipeBuildJob`."""
+
+ def __init__(self, craft_recipe_build_job):
+ self.context = craft_recipe_build_job
+
+ def __repr__(self):
+ """An informative representation of the job."""
+ recipe = self.build.recipe
+ return "<%s for ~%s/%s/+craft/%s/+build/%d>" % (
+ self.__class__.__name__,
+ recipe.owner.name,
+ recipe.project.name,
+ recipe.name,
+ self.build.id,
+ )
+
+ @classmethod
+ def get(cls, job_id):
+ craft_recipe_build_job = IStore(CraftRecipeBuildJob).get(
+ CraftRecipeBuildJob, job_id
+ )
+ if craft_recipe_build_job.job_type != cls.class_job_type:
+ raise NotFoundError(
+ "No object found with id %d and type %s"
+ % (job_id, cls.class_job_type.title)
+ )
+ return cls(craft_recipe_build_job)
+
+ @classmethod
+ def iterReady(cls):
+ """See `IJobSource`."""
+ jobs = IPrimaryStore(CraftRecipeBuildJob).find(
+ CraftRecipeBuildJob,
+ CraftRecipeBuildJob.job_type == cls.class_job_type,
+ CraftRecipeBuildJob.job == Job.id,
+ Job.id.is_in(Job.ready_jobs),
+ )
+ return (cls(job) for job in jobs)
+
+ def getOopsVars(self):
+ """See `IRunnableJob`."""
+ oops_vars = super().getOopsVars()
+ recipe = self.context.build.recipe
+ oops_vars.extend(
+ [
+ ("job_id", self.context.job.id),
+ ("job_type", self.context.job_type.title),
+ ("build_id", self.context.build.id),
+ ("recipe_owner_name", recipe.owner.name),
+ ("recipe_project_name", recipe.project.name),
+ ("recipe_name", recipe.name),
+ ]
+ )
+ return oops_vars
+
+
+@implementer(ICraftPublishingJob)
+@provider(ICraftPublishingJobSource)
+class CraftPublishingJob(CraftRecipeBuildJobDerived):
+ """
+ A Job that publishes craft recipe build artifacts to external
+ repositories.
+ """
+
+ class_job_type = CraftRecipeBuildJobType.PUBLISH_ARTIFACTS
+
+ user_error_types = ()
+ retry_error_types = ()
+ max_retries = 5
+
+ task_queue = "native_publisher_job"
+
+ config = config.ICraftPublishingJobSource
+
+ @classmethod
+ def create(cls, build):
+ """See `ICraftPublishingJobSource`."""
+ metadata = {
+ "build_id": build.id,
+ }
+ recipe_job = CraftRecipeBuildJob(build, cls.class_job_type, metadata)
+ job = cls(recipe_job)
+ job.celeryRunOnCommit()
+ IStore(CraftRecipeBuildJob).flush()
+ return job
+
+ @property
+ def error_message(self):
+ """See `ICraftPublishingJob`."""
+ return self.metadata.get("error_message")
+
+ @error_message.setter
+ def error_message(self, message):
+ """See `ICraftPublishingJob`."""
+ self.metadata["error_message"] = message
+
+ def run(self):
+ """See `IRunnableJob`."""
+ try:
+ # Get the distribution name to access the correct configuration
+ distribution_name = None
+ git_repo = self.build.recipe.git_repository
+ if git_repo is not None:
+ if IDistributionSourcePackage.providedBy(git_repo.target):
+ distribution_name = git_repo.target.distribution.name
+
+ if not distribution_name:
+ self.error_message = (
+ "Could not determine distribution for build"
+ )
+ raise Exception(self.error_message)
+
+ # Get environment variables from configuration
+ try:
+ env_vars_json = config["craftbuild." + distribution_name][
+ "environment_variables"
+ ]
+ if env_vars_json and env_vars_json.lower() != "none":
+ env_vars = json.loads(env_vars_json)
+ # Replace auth placeholders
+ for key, value in env_vars.items():
+ if (
+ isinstance(value, str)
+ and "%(write_auth)s" in value
+ ):
+ env_vars[key] = value.replace(
+ "%(write_auth)s",
+ config.artifactory.write_credentials,
+ )
+ else:
+ env_vars = {}
+ except (NoSectionError, KeyError):
+ self.error_message = (
+ f"No configuration found for {distribution_name}"
+ )
+ raise Exception(self.error_message)
+
+ # Check if we have a .crate file or .jar file
+ crate_file = None
+ jar_file = None
+ pom_file = None
+
+ for _, lfa, _ in self.build.getFiles():
+ if lfa.filename.endswith(".crate"):
+ crate_file = lfa
+ elif lfa.filename.endswith(".jar"):
+ jar_file = lfa
+ elif lfa.filename == "pom.xml":
+ pom_file = lfa
+
+ # Process the crate file
+ with tempfile.TemporaryDirectory() as tmpdir:
+ if crate_file is not None:
+ # Download the crate file
+ crate_path = os.path.join(tmpdir, crate_file.filename)
+ crate_file.open()
+ try:
+ with open(crate_path, "wb") as f:
+ f.write(crate_file.read())
+ finally:
+ crate_file.close()
+
+ # Create a directory to extract the crate
+ crate_extract_dir = os.path.join(tmpdir, "crate_contents")
+ os.makedirs(crate_extract_dir, exist_ok=True)
+
+ # Extract the .crate file using system tar command
+ result = subprocess.run(
+ ["tar", "-xf", crate_path, "-C", crate_extract_dir],
+ capture_output=True,
+ text=True,
+ )
+
+ if result.returncode != 0:
+ raise Exception(
+ f"Failed to extract crate: {result.stderr}"
+ )
+
+ # Find the extracted directory(should be the only one)
+ extracted_dirs = [
+ d
+ for d in os.listdir(crate_extract_dir)
+ if os.path.isdir(os.path.join(crate_extract_dir, d))
+ ]
+
+ if not extracted_dirs:
+ raise Exception(
+ "No directory found in extracted crate"
+ )
+
+ # Use the first directory as the crate directory
+ crate_dir = os.path.join(
+ crate_extract_dir, extracted_dirs[0]
+ )
+
+ # Publish the Rust crate
+ self._publish_rust_crate(crate_dir, env_vars)
+ elif jar_file is not None and pom_file is not None:
+ # Download the jar file
+ jar_path = os.path.join(tmpdir, jar_file.filename)
+ jar_file.open()
+ try:
+ with open(jar_path, "wb") as f:
+ f.write(jar_file.read())
+ finally:
+ jar_file.close()
+
+ # Download the pom file
+ pom_path = os.path.join(tmpdir, "pom.xml")
+ pom_file.open()
+ try:
+ with open(pom_path, "wb") as f:
+ f.write(pom_file.read())
+ finally:
+ pom_file.close()
+
+ # Publish the Maven artifact
+ self._publish_maven_artifact(
+ tmpdir,
+ env_vars,
+ jar_path,
+ pom_path,
+ )
+
+ else:
+ raise Exception("No publishable artifacts found in build")
+
+ except Exception as e:
+ self.error_message = str(e)
+ # The normal job infrastructure will abort the transaction, but
+ # we want to commit instead: the only database changes we make
+ # are to this job's metadata and should be preserved.
+ transaction.commit()
+ raise
+
+ def _publish_rust_crate(self, extract_dir, env_vars):
+ """Publish Rust crates from the extracted crate directory.
+
+ :param extract_dir: Path to the extracted crate directory
+ :param env_vars: Environment variables from configuration
+ :raises: Exception if publishing fails
+ """
+ # Look for specific Cargo publishing repository configuration
+ cargo_publish_url = env_vars.get("CARGO_PUBLISH_URL")
+ cargo_publish_auth = env_vars.get("CARGO_PUBLISH_AUTH")
+
+ if not cargo_publish_url or not cargo_publish_auth:
+ raise Exception(
+ "Missing Cargo publishing repository configuration"
+ )
+
+ # Extract token from auth string (discard username if present)
+ if ":" in cargo_publish_auth:
+ _, token = cargo_publish_auth.split(":", 1)
+ else:
+ token = cargo_publish_auth
+
+ # Set up cargo config
+ cargo_dir = os.path.join(extract_dir, ".cargo")
+ os.makedirs(cargo_dir, exist_ok=True)
+
+ # Create config.toml
+ with open(os.path.join(cargo_dir, "config.toml"), "w") as f:
+ f.write(
+ "\n"
+ "[registry]\n"
+ 'global-credential-providers = ["cargo:token"]\n'
+ "\n"
+ "[registries.launchpad]\n"
+ f'index = "{cargo_publish_url}"\n'
+ )
+
+ # Create credentials.toml
+ with open(os.path.join(cargo_dir, "credentials.toml"), "w") as f:
+ f.write(
+ "\n" "[registries.launchpad]\n" f'token = "Bearer {token}"\n'
+ )
+
+ # Run cargo publish from the extracted directory
+ result = subprocess.run(
+ [
+ "cargo",
+ "publish",
+ "--no-verify",
+ "--allow-dirty",
+ "--registry",
+ "launchpad",
+ ],
+ capture_output=True,
+ cwd=extract_dir,
+ env={"CARGO_HOME": cargo_dir},
+ )
+
+ if result.returncode != 0:
+ raise Exception(f"Failed to publish crate: {result.stderr}")
+
+ def _publish_maven_artifact(
+ self, work_dir, env_vars, jar_path=None, pom_path=None
+ ):
+ """Publish Maven artifacts.
+
+ :param work_dir: Working directory
+ :param env_vars: Environment variables from configuration
+ :param jar_path: Path to the JAR file
+ :param pom_path: Path to the pom.xml file
+ :raises: Exception if publishing fails
+ """
+ # Look for specific Maven publishing repository configuration
+ maven_publish_url = env_vars.get("MAVEN_PUBLISH_URL")
+ maven_publish_auth = env_vars.get("MAVEN_PUBLISH_AUTH")
+
+ if not maven_publish_url or not maven_publish_auth:
+ raise Exception(
+ "Missing Maven publishing repository configuration"
+ )
+
+ if jar_path is None or pom_path is None:
+ raise Exception("Missing JAR or POM file for Maven publishing")
+
+ # Set up Maven settings
+ maven_dir = os.path.join(work_dir, ".m2")
+ os.makedirs(maven_dir, exist_ok=True)
+
+ # Extract username and password from auth string
+ if ":" in maven_publish_auth:
+ username, password = maven_publish_auth.split(":", 1)
+ else:
+ username = "launchpad"
+ password = maven_publish_auth
+
+ # Generate settings.xml content
+ settings_xml = self._get_maven_settings_xml(username, password)
+
+ with open(os.path.join(maven_dir, "settings.xml"), "w") as f:
+ f.write(settings_xml)
+
+ # Run mvn deploy using the pom file
+ result = subprocess.run(
+ [
+ "mvn",
+ "deploy:deploy-file",
+ f"-DpomFile={pom_path}",
+ f"-Dfile={jar_path}",
+ "-DrepositoryId=central",
+ f"-Durl={maven_publish_url}",
+ "--settings={}".format(
+ os.path.join(maven_dir, "settings.xml")
+ ),
+ ],
+ capture_output=True,
+ cwd=work_dir,
+ )
+
+ if result.returncode != 0:
+ raise Exception(
+ f"Failed to publish Maven artifact: {result.stderr}"
+ )
+
+ def _get_maven_settings_xml(self, username, password):
+ """Generate Maven settings.xml content.
+
+ :param username: Maven repository username
+ :param password: Maven repository password
+ :return: XML content as string
+ """
+ # Break it into smaller parts to avoid long lines
+ header = (
+ '<?xml version="1.0" encoding="UTF-8"?>\n'
+ '<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"\n'
+ ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"\n'
+ )
+
+ schema = (
+ ' xsi:schemaLocation="'
+ "http://maven.apache.org/SETTINGS/1.0.0 "
+ 'http://maven.apache.org/xsd/settings-1.0.0.xsd";>\n'
+ )
+
+ servers = (
+ " <servers>\n"
+ " <server>\n"
+ " <id>central</id>\n"
+ f" <username>{username}</username>\n"
+ f" <password>{password}</password>\n"
+ " </server>\n"
+ " </servers>\n"
+ )
+
+ profiles = (
+ " <profiles>\n"
+ " <profile>\n"
+ " <id>system</id>\n"
+ " <pluginRepositories>\n"
+ " <pluginRepository>\n"
+ " <id>central</id>\n"
+ " <url>file:///usr/share/maven-repo</url>\n"
+ " </pluginRepository>\n"
+ " </pluginRepositories>\n"
+ " </profile>\n"
+ " </profiles>\n"
+ )
+
+ active_profiles = (
+ " <activeProfiles>\n"
+ " <activeProfile>system</activeProfile>\n"
+ " </activeProfiles>\n"
+ "</settings>"
+ )
+
+ # Combine all parts
+ return header + schema + servers + profiles + active_profiles
diff --git a/lib/lp/crafts/model/craftrecipejob.py b/lib/lp/crafts/model/craftrecipejob.py
index 40d3104..d573665 100644
--- a/lib/lp/crafts/model/craftrecipejob.py
+++ b/lib/lp/crafts/model/craftrecipejob.py
@@ -7,15 +7,8 @@ __all__ = [
"CraftRecipeJob",
"CraftRecipeJobType",
"CraftRecipeRequestBuildsJob",
- "CraftPublishingJob",
]
-import json
-import os
-import subprocess
-import tempfile
-from configparser import NoSectionError
-
import transaction
from lazr.delegates import delegate_to
from lazr.enum import DBEnumeratedType, DBItem
@@ -32,16 +25,11 @@ from lp.crafts.interfaces.craftrecipe import (
MissingSourcecraftYaml,
)
from lp.crafts.interfaces.craftrecipejob import (
- ICraftPublishingJob,
- ICraftPublishingJobSource,
ICraftRecipeJob,
ICraftRecipeRequestBuildsJob,
ICraftRecipeRequestBuildsJobSource,
)
from lp.crafts.model.craftrecipebuild import CraftRecipeBuild
-from lp.registry.interfaces.distributionsourcepackage import (
- IDistributionSourcePackage,
-)
from lp.registry.interfaces.person import IPersonSet
from lp.services.config import config
from lp.services.database.bulk import load_related
@@ -68,16 +56,6 @@ class CraftRecipeJobType(DBEnumeratedType):
""",
)
- PUBLISH_ARTIFACTS = DBItem(
- 1,
- """
- Publish artifacts
-
- This job publishes craft recipe build artifacts to external
- repositories.
- """,
- )
-
@implementer(ICraftRecipeJob)
class CraftRecipeJob(StormBase):
@@ -359,370 +337,3 @@ class CraftRecipeRequestBuildsJob(CraftRecipeJobDerived):
# are to this job's metadata and should be preserved.
transaction.commit()
raise
-
-
-@implementer(ICraftPublishingJob)
-@provider(ICraftPublishingJobSource)
-class CraftPublishingJob(CraftRecipeJobDerived):
- """
- A Job that publishes craft recipe build artifacts to external
- repositories.
- """
-
- class_job_type = CraftRecipeJobType.PUBLISH_ARTIFACTS
-
- user_error_types = ()
- retry_error_types = ()
- max_retries = 5
-
- task_queue = "native_publisher_job"
-
- config = config.ICraftPublishingJobSource
-
- @classmethod
- def create(cls, build):
- """See `ICraftPublishingJobSource`."""
- metadata = {
- "build_id": build.id,
- }
- recipe_job = CraftRecipeJob(build.recipe, cls.class_job_type, metadata)
- job = cls(recipe_job)
- job.celeryRunOnCommit()
- IStore(CraftRecipeJob).flush()
- return job
-
- @property
- def build_id(self):
- """See `ICraftPublishingJob`."""
- return self.metadata["build_id"]
-
- @cachedproperty
- def build(self):
- """See `ICraftPublishingJob`."""
- return IStore(CraftRecipeBuild).get(CraftRecipeBuild, self.build_id)
-
- @property
- def error_message(self):
- """See `ICraftPublishingJob`."""
- return self.metadata.get("error_message")
-
- @error_message.setter
- def error_message(self, message):
- """See `ICraftPublishingJob`."""
- self.metadata["error_message"] = message
-
- def run(self):
- """See `IRunnableJob`."""
- try:
- # Get the distribution name to access the correct configuration
- distribution_name = None
- git_repo = self.build.recipe.git_repository
- if git_repo is not None:
- if IDistributionSourcePackage.providedBy(git_repo.target):
- distribution_name = git_repo.target.distribution.name
-
- if not distribution_name:
- self.error_message = (
- "Could not determine distribution for build"
- )
- raise Exception(self.error_message)
-
- # Get environment variables from configuration
- try:
- env_vars_json = config["craftbuild." + distribution_name][
- "environment_variables"
- ]
- if env_vars_json and env_vars_json.lower() != "none":
- env_vars = json.loads(env_vars_json)
- # Replace auth placeholders
- for key, value in env_vars.items():
- if (
- isinstance(value, str)
- and "%(write_auth)s" in value
- ):
- env_vars[key] = value.replace(
- "%(write_auth)s",
- config.artifactory.write_credentials,
- )
- else:
- env_vars = {}
- except (NoSectionError, KeyError):
- self.error_message = (
- f"No configuration found for {distribution_name}"
- )
- raise Exception(self.error_message)
-
- # Check if we have a .crate file or .jar file
- crate_file = None
- jar_file = None
- pom_file = None
-
- for _, lfa, _ in self.build.getFiles():
- if lfa.filename.endswith(".crate"):
- crate_file = lfa
- elif lfa.filename.endswith(".jar"):
- jar_file = lfa
- elif lfa.filename == "pom.xml":
- pom_file = lfa
-
- # Process the crate file
- with tempfile.TemporaryDirectory() as tmpdir:
- if crate_file is not None:
- # Download the crate file
- crate_path = os.path.join(tmpdir, crate_file.filename)
- crate_file.open()
- try:
- with open(crate_path, "wb") as f:
- f.write(crate_file.read())
- finally:
- crate_file.close()
-
- # Create a directory to extract the crate
- crate_extract_dir = os.path.join(tmpdir, "crate_contents")
- os.makedirs(crate_extract_dir, exist_ok=True)
-
- # Extract the .crate file using system tar command
- result = subprocess.run(
- ["tar", "-xf", crate_path, "-C", crate_extract_dir],
- capture_output=True,
- text=True,
- )
-
- if result.returncode != 0:
- raise Exception(
- f"Failed to extract crate: {result.stderr}"
- )
-
- # Find the extracted directory(should be the only one)
- extracted_dirs = [
- d
- for d in os.listdir(crate_extract_dir)
- if os.path.isdir(os.path.join(crate_extract_dir, d))
- ]
-
- if not extracted_dirs:
- raise Exception(
- "No directory found in extracted crate"
- )
-
- # Use the first directory as the crate directory
- crate_dir = os.path.join(
- crate_extract_dir, extracted_dirs[0]
- )
-
- # Publish the Rust crate
- self._publish_rust_crate(crate_dir, env_vars)
- elif jar_file is not None and pom_file is not None:
- # Download the jar file
- jar_path = os.path.join(tmpdir, jar_file.filename)
- jar_file.open()
- try:
- with open(jar_path, "wb") as f:
- f.write(jar_file.read())
- finally:
- jar_file.close()
-
- # Download the pom file
- pom_path = os.path.join(tmpdir, "pom.xml")
- pom_file.open()
- try:
- with open(pom_path, "wb") as f:
- f.write(pom_file.read())
- finally:
- pom_file.close()
-
- # Publish the Maven artifact
- self._publish_maven_artifact(
- tmpdir,
- env_vars,
- jar_path,
- pom_path,
- )
-
- else:
- raise Exception("No publishable artifacts found in build")
-
- except Exception as e:
- self.error_message = str(e)
- # The normal job infrastructure will abort the transaction, but
- # we want to commit instead: the only database changes we make
- # are to this job's metadata and should be preserved.
- transaction.commit()
- raise
-
- def _publish_rust_crate(self, extract_dir, env_vars):
- """Publish Rust crates from the extracted crate directory.
-
- :param extract_dir: Path to the extracted crate directory
- :param env_vars: Environment variables from configuration
- :raises: Exception if publishing fails
- """
- # Look for specific Cargo publishing repository configuration
- cargo_publish_url = env_vars.get("CARGO_PUBLISH_URL")
- cargo_publish_auth = env_vars.get("CARGO_PUBLISH_AUTH")
-
- if not cargo_publish_url or not cargo_publish_auth:
- raise Exception(
- "Missing Cargo publishing repository configuration"
- )
-
- # Extract token from auth string (discard username if present)
- if ":" in cargo_publish_auth:
- _, token = cargo_publish_auth.split(":", 1)
- else:
- token = cargo_publish_auth
-
- # Set up cargo config
- cargo_dir = os.path.join(extract_dir, ".cargo")
- os.makedirs(cargo_dir, exist_ok=True)
-
- # Create config.toml
- with open(os.path.join(cargo_dir, "config.toml"), "w") as f:
- f.write(
- "\n"
- "[registry]\n"
- 'global-credential-providers = ["cargo:token"]\n'
- "\n"
- "[registries.launchpad]\n"
- f'index = "{cargo_publish_url}"\n'
- )
-
- # Create credentials.toml
- with open(os.path.join(cargo_dir, "credentials.toml"), "w") as f:
- f.write(
- "\n" "[registries.launchpad]\n" f'token = "Bearer {token}"\n'
- )
-
- # Run cargo publish from the extracted directory
- result = subprocess.run(
- [
- "cargo",
- "publish",
- "--no-verify",
- "--allow-dirty",
- "--registry",
- "launchpad",
- ],
- capture_output=True,
- cwd=extract_dir,
- env={"CARGO_HOME": cargo_dir},
- )
-
- if result.returncode != 0:
- raise Exception(f"Failed to publish crate: {result.stderr}")
-
- def _publish_maven_artifact(
- self, work_dir, env_vars, jar_path=None, pom_path=None
- ):
- """Publish Maven artifacts.
-
- :param work_dir: Working directory
- :param env_vars: Environment variables from configuration
- :param jar_path: Path to the JAR file
- :param pom_path: Path to the pom.xml file
- :raises: Exception if publishing fails
- """
- # Look for specific Maven publishing repository configuration
- maven_publish_url = env_vars.get("MAVEN_PUBLISH_URL")
- maven_publish_auth = env_vars.get("MAVEN_PUBLISH_AUTH")
-
- if not maven_publish_url or not maven_publish_auth:
- raise Exception(
- "Missing Maven publishing repository configuration"
- )
-
- if jar_path is None or pom_path is None:
- raise Exception("Missing JAR or POM file for Maven publishing")
-
- # Set up Maven settings
- maven_dir = os.path.join(work_dir, ".m2")
- os.makedirs(maven_dir, exist_ok=True)
-
- # Extract username and password from auth string
- if ":" in maven_publish_auth:
- username, password = maven_publish_auth.split(":", 1)
- else:
- username = "launchpad"
- password = maven_publish_auth
-
- # Generate settings.xml content
- settings_xml = self._get_maven_settings_xml(username, password)
-
- with open(os.path.join(maven_dir, "settings.xml"), "w") as f:
- f.write(settings_xml)
-
- # Run mvn deploy using the pom file
- result = subprocess.run(
- [
- "mvn",
- "deploy:deploy-file",
- f"-DpomFile={pom_path}",
- f"-Dfile={jar_path}",
- "-DrepositoryId=central",
- f"-Durl={maven_publish_url}",
- "--settings={}".format(
- os.path.join(maven_dir, "settings.xml")
- ),
- ],
- capture_output=True,
- cwd=work_dir,
- )
-
- if result.returncode != 0:
- raise Exception(
- f"Failed to publish Maven artifact: {result.stderr}"
- )
-
- def _get_maven_settings_xml(self, username, password):
- """Generate Maven settings.xml content.
-
- :param username: Maven repository username
- :param password: Maven repository password
- :return: XML content as string
- """
- # Break it into smaller parts to avoid long lines
- header = (
- '<?xml version="1.0" encoding="UTF-8"?>\n'
- '<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"\n'
- ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"\n'
- )
-
- schema = (
- ' xsi:schemaLocation="'
- "http://maven.apache.org/SETTINGS/1.0.0 "
- 'http://maven.apache.org/xsd/settings-1.0.0.xsd";>\n'
- )
-
- servers = (
- " <servers>\n"
- " <server>\n"
- " <id>central</id>\n"
- f" <username>{username}</username>\n"
- f" <password>{password}</password>\n"
- " </server>\n"
- " </servers>\n"
- )
-
- profiles = (
- " <profiles>\n"
- " <profile>\n"
- " <id>system</id>\n"
- " <pluginRepositories>\n"
- " <pluginRepository>\n"
- " <id>central</id>\n"
- " <url>file:///usr/share/maven-repo</url>\n"
- " </pluginRepository>\n"
- " </pluginRepositories>\n"
- " </profile>\n"
- " </profiles>\n"
- )
-
- active_profiles = (
- " <activeProfiles>\n"
- " <activeProfile>system</activeProfile>\n"
- " </activeProfiles>\n"
- "</settings>"
- )
-
- # Combine all parts
- return header + schema + servers + profiles + active_profiles
diff --git a/lib/lp/crafts/subscribers/craftrecipebuild.py b/lib/lp/crafts/subscribers/craftrecipebuild.py
index 5b29d9d..5437c55 100644
--- a/lib/lp/crafts/subscribers/craftrecipebuild.py
+++ b/lib/lp/crafts/subscribers/craftrecipebuild.py
@@ -8,7 +8,7 @@ from zope.component import getUtility
from lp.buildmaster.enums import BuildStatus
from lp.crafts.interfaces.craftrecipebuild import ICraftRecipeBuild
-from lp.crafts.interfaces.craftrecipejob import ICraftPublishingJobSource
+from lp.crafts.interfaces.craftrecipebuildjob import ICraftPublishingJobSource
from lp.registry.interfaces.distributionsourcepackage import (
IDistributionSourcePackage,
)
diff --git a/lib/lp/crafts/tests/test_craftrecipebuildjob.py b/lib/lp/crafts/tests/test_craftrecipebuildjob.py
new file mode 100644
index 0000000..8bd5155
--- /dev/null
+++ b/lib/lp/crafts/tests/test_craftrecipebuildjob.py
@@ -0,0 +1,415 @@
+# Copyright 2025 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+import io
+import json
+import tarfile
+
+from fixtures import FakeLogger
+from testtools.matchers import Equals, Is, MatchesStructure
+from zope.component import getUtility
+from zope.security.proxy import removeSecurityProxy
+
+from lp.crafts.interfaces.craftrecipe import CRAFT_RECIPE_ALLOW_CREATE
+from lp.crafts.interfaces.craftrecipebuildjob import (
+ ICraftPublishingJob,
+ ICraftPublishingJobSource,
+ ICraftRecipeBuildJob,
+)
+from lp.crafts.model.craftrecipebuildjob import (
+ CraftPublishingJob,
+ CraftRecipeBuildJob,
+ CraftRecipeBuildJobType,
+)
+from lp.services.features.testing import FeatureFixture
+from lp.services.job.interfaces.job import JobStatus
+from lp.services.job.runner import JobRunner
+from lp.services.librarian.interfaces import ILibraryFileAliasSet
+from lp.testing import TestCaseWithFactory
+from lp.testing.layers import CeleryJobLayer, ZopelessDatabaseLayer
+
+
+class TestCraftRecipeBuildJob(TestCaseWithFactory):
+
+ layer = ZopelessDatabaseLayer
+
+ def setUp(self):
+ super().setUp()
+ self.useFixture(FeatureFixture({CRAFT_RECIPE_ALLOW_CREATE: "on"}))
+
+ def test_provides_interface(self):
+ # CraftRecipeBuildJob provides ICraftRecipeBuildJob.
+ build = self.factory.makeCraftRecipeBuild()
+ self.assertProvides(
+ CraftRecipeBuildJob(
+ build,
+ CraftRecipeBuildJobType.PUBLISH_ARTIFACTS,
+ {},
+ ),
+ ICraftRecipeBuildJob,
+ )
+
+
+class TestCraftPublishingJob(TestCaseWithFactory):
+ """Test the CraftPublishingJob."""
+
+ layer = CeleryJobLayer
+
+ def setUp(self):
+ super().setUp()
+ self.useFixture(
+ FeatureFixture(
+ {"jobs.celery.enabled_classes": "CraftPublishingJob"}
+ )
+ )
+ self.useFixture(FakeLogger())
+ self.useFixture(FeatureFixture({CRAFT_RECIPE_ALLOW_CREATE: "on"}))
+ self.recipe = self.factory.makeCraftRecipe()
+ self.build = self.factory.makeCraftRecipeBuild(recipe=self.recipe)
+
+ def test_provides_interface(self):
+ # CraftPublishingJob provides ICraftPublishingJob.
+ job = getUtility(ICraftPublishingJobSource).create(self.build)
+
+ # Check that the instance provides the job interface
+ self.assertProvides(job, ICraftPublishingJob)
+
+ # Check that the class provides the source interface
+ self.assertProvides(CraftPublishingJob, ICraftPublishingJobSource)
+
+ def test_create(self):
+ # CraftPublishingJob.create creates a CraftPublishingJob with the
+ # correct attributes.
+ job = getUtility(ICraftPublishingJobSource).create(self.build)
+
+ job = removeSecurityProxy(job)
+
+ self.assertThat(
+ job,
+ MatchesStructure(
+ class_job_type=Equals(
+ CraftRecipeBuildJobType.PUBLISH_ARTIFACTS
+ ),
+ build=Equals(self.build),
+ error_message=Is(None),
+ ),
+ )
+
+ def test_run_failure_cannot_determine_distribution(self):
+ """Test failure when distribution cannot be determined."""
+ job = getUtility(ICraftPublishingJobSource).create(self.build)
+
+ # Create a mock git repository with a non-distribution target
+ git_repo = self.factory.makeGitRepository()
+ self.patch(
+ removeSecurityProxy(self.recipe), "git_repository", git_repo
+ )
+
+ JobRunner([job]).runAll()
+
+ job = removeSecurityProxy(job)
+ self.assertEqual(JobStatus.FAILED, job.job.status)
+ self.assertEqual(
+ "Could not determine distribution for build",
+ job.error_message,
+ )
+
+ def test_run_failure_no_configuration(self):
+ """Test failure when no configuration is found for the distribution."""
+ # Create a distribution with a name that won't have a configuration
+ distribution = self.factory.makeDistribution(name="nonexistent")
+
+ # Create a distribution source package for our distribution
+ package = self.factory.makeDistributionSourcePackage(
+ distribution=distribution
+ )
+
+ # Create a git repository targeting that package
+ git_repository = self.factory.makeGitRepository(target=package)
+
+ # Update our recipe to use this git repository
+ removeSecurityProxy(self.recipe).git_repository = git_repository
+
+ job = getUtility(ICraftPublishingJobSource).create(self.build)
+ JobRunner([job]).runAll()
+ job = removeSecurityProxy(job)
+
+ self.assertEqual(JobStatus.FAILED, job.job.status)
+ self.assertEqual(
+ "No configuration found for nonexistent",
+ job.error_message,
+ )
+
+ def test_run_no_publishable_artifacts(self):
+ """Test failure when no publishable artifacts are found."""
+ distribution = self.factory.makeDistribution(name="soss")
+
+ # Set up config with environment variables but no Cargo publishing info
+ # We just need a config section for the distribution name
+ self.pushConfig("craftbuild.soss")
+ package = self.factory.makeDistributionSourcePackage(
+ distribution=distribution
+ )
+ git_repository = self.factory.makeGitRepository(target=package)
+ removeSecurityProxy(self.recipe).git_repository = git_repository
+
+ # Create a dummy file (but not a crate or jar)
+ from io import BytesIO
+
+ dummy_content = b"test content"
+
+ # Create a LibraryFileAlias with the dummy content
+ librarian = getUtility(ILibraryFileAliasSet)
+ lfa = librarian.create(
+ "test.txt",
+ len(dummy_content),
+ BytesIO(dummy_content),
+ "text/plain",
+ )
+
+ # Add the file to the build
+ removeSecurityProxy(self.build).addFile(lfa)
+
+ job = getUtility(ICraftPublishingJobSource).create(self.build)
+ JobRunner([job]).runAll()
+ job = removeSecurityProxy(job)
+ self.assertEqual(JobStatus.FAILED, job.job.status)
+
+ self.assertEqual(
+ "No publishable artifacts found in build",
+ job.error_message,
+ )
+
+ def test_run_missing_cargo_config(self):
+ """Test failure when a crate is found but Cargo config is missing."""
+ distribution = self.factory.makeDistribution(name="soss")
+ self.pushConfig("craftbuild.soss")
+ package = self.factory.makeDistributionSourcePackage(
+ distribution=distribution
+ )
+
+ git_repository = self.factory.makeGitRepository(target=package)
+
+ removeSecurityProxy(self.recipe).git_repository = git_repository
+
+ # Create a BytesIO object to hold the tar data
+ tar_data = io.BytesIO()
+
+ # Create a tar archive
+ with tarfile.open(fileobj=tar_data, mode="w") as tar:
+ # Create a directory entry for the crate
+ crate_dir_info = tarfile.TarInfo("test-0.1.0")
+ crate_dir_info.type = tarfile.DIRTYPE
+ tar.addfile(crate_dir_info)
+
+ # Add a Cargo.toml file
+ cargo_toml = """
+[package]
+name = "test"
+version = "0.1.0"
+authors = ["Test <test@xxxxxxxxxxx>"]
+edition = "2018"
+"""
+ cargo_toml_info = tarfile.TarInfo("test-0.1.0/Cargo.toml")
+ cargo_toml_bytes = cargo_toml.encode("utf-8")
+ cargo_toml_info.size = len(cargo_toml_bytes)
+ tar.addfile(cargo_toml_info, io.BytesIO(cargo_toml_bytes))
+
+ # Add a src directory
+ src_dir_info = tarfile.TarInfo("test-0.1.0/src")
+ src_dir_info.type = tarfile.DIRTYPE
+ tar.addfile(src_dir_info)
+
+ # Add a main.rs file
+ main_rs = 'fn main() { println!("Hello, world!"); }'
+ main_rs_info = tarfile.TarInfo("test-0.1.0/src/main.rs")
+ main_rs_bytes = main_rs.encode("utf-8")
+ main_rs_info.size = len(main_rs_bytes)
+ tar.addfile(main_rs_info, io.BytesIO(main_rs_bytes))
+
+ # Get the tar data
+ tar_data.seek(0)
+ crate_content = tar_data.getvalue()
+
+ # Create a LibraryFileAlias with the crate content
+ librarian = getUtility(ILibraryFileAliasSet)
+ lfa = librarian.create(
+ "test-0.1.0.crate",
+ len(crate_content),
+ io.BytesIO(crate_content),
+ "application/x-tar",
+ )
+
+ removeSecurityProxy(self.build).addFile(lfa)
+
+ job = getUtility(ICraftPublishingJobSource).create(self.build)
+ JobRunner([job]).runAll()
+ job = removeSecurityProxy(job)
+
+ self.assertEqual(JobStatus.FAILED, job.job.status)
+ self.assertEqual(
+ "Missing Cargo publishing repository configuration",
+ job.error_message,
+ )
+
+ def test_run_crate_extraction_failure(self):
+ """Test failure when crate extraction fails."""
+ distribution = self.factory.makeDistribution(name="soss")
+
+ # Set up config with environment variables
+ self.pushConfig(
+ "craftbuild.soss",
+ environment_variables=json.dumps(
+ {
+ "CARGO_PUBLISH_URL": "https://example.com/registry";,
+ "CARGO_PUBLISH_AUTH": "lp:token123",
+ }
+ ),
+ )
+
+ package = self.factory.makeDistributionSourcePackage(
+ distribution=distribution
+ )
+
+ git_repository = self.factory.makeGitRepository(target=package)
+
+ removeSecurityProxy(self.recipe).git_repository = git_repository
+
+ # Create an invalid tar file (just some random bytes)
+ invalid_crate_content = b"This is not a valid tar file"
+
+ librarian = getUtility(ILibraryFileAliasSet)
+ lfa = librarian.create(
+ "invalid-0.1.0.crate",
+ len(invalid_crate_content),
+ io.BytesIO(invalid_crate_content),
+ "application/x-tar",
+ )
+
+ removeSecurityProxy(self.build).addFile(lfa)
+
+ job = getUtility(ICraftPublishingJobSource).create(self.build)
+ JobRunner([job]).runAll()
+ job = removeSecurityProxy(job)
+
+ self.assertEqual(JobStatus.FAILED, job.job.status)
+ self.assertIn(
+ "Failed to extract crate",
+ job.error_message,
+ )
+
+ def test_run_no_directory_in_crate(self):
+ """Test failure when no directory is found in extracted crate."""
+ distribution = self.factory.makeDistribution(name="soss")
+
+ # Set up config with environment variables
+ self.pushConfig(
+ "craftbuild.soss",
+ environment_variables=json.dumps(
+ {
+ "CARGO_PUBLISH_URL": "https://example.com/registry";,
+ "CARGO_PUBLISH_AUTH": "lp:token123",
+ }
+ ),
+ )
+
+ package = self.factory.makeDistributionSourcePackage(
+ distribution=distribution
+ )
+
+ git_repository = self.factory.makeGitRepository(target=package)
+
+ removeSecurityProxy(self.recipe).git_repository = git_repository
+
+ tar_data = io.BytesIO()
+
+ # Create a tar archive with only files (no directories)
+ with tarfile.open(fileobj=tar_data, mode="w") as tar:
+ # Add a file at the root level (no directory)
+ file_info = tarfile.TarInfo("file.txt")
+ file_content = b"This is a file with no directory"
+ file_info.size = len(file_content)
+ tar.addfile(file_info, io.BytesIO(file_content))
+
+ # Get the tar data
+ tar_data.seek(0)
+ crate_content = tar_data.getvalue()
+
+ # Create a LibraryFileAlias with the crate content
+ librarian = getUtility(ILibraryFileAliasSet)
+ lfa = librarian.create(
+ "nodirs-0.1.0.crate",
+ len(crate_content),
+ io.BytesIO(crate_content),
+ "application/x-tar",
+ )
+
+ # Add the file to the build
+ removeSecurityProxy(self.build).addFile(lfa)
+
+ # Create and run the job
+ job = getUtility(ICraftPublishingJobSource).create(self.build)
+
+ JobRunner([job]).runAll()
+
+ # Verify job failed with expected error message
+ job = removeSecurityProxy(job)
+ self.assertEqual(JobStatus.FAILED, job.job.status)
+ self.assertEqual(
+ "No directory found in extracted crate",
+ job.error_message,
+ )
+
+ def test_run_missing_maven_config(self):
+ """
+ Test failure when Maven artifacts are found but Maven config is
+ missing.
+ """
+ distribution = self.factory.makeDistribution(name="soss")
+
+ self.pushConfig("craftbuild.soss")
+ package = self.factory.makeDistributionSourcePackage(
+ distribution=distribution
+ )
+ git_repository = self.factory.makeGitRepository(target=package)
+ removeSecurityProxy(self.recipe).git_repository = git_repository
+
+ # Create a dummy jar file
+ from io import BytesIO
+
+ dummy_jar_content = b"dummy jar content"
+
+ # Create a LibraryFileAlias with the jar content
+ librarian = getUtility(ILibraryFileAliasSet)
+ jar_lfa = librarian.create(
+ "test-0.1.0.jar",
+ len(dummy_jar_content),
+ BytesIO(dummy_jar_content),
+ "application/java-archive",
+ )
+
+ # Create a dummy pom file
+ dummy_pom_content = b"<project>...</project>"
+
+ # Create a LibraryFileAlias with the pom content
+ pom_lfa = librarian.create(
+ "pom.xml",
+ len(dummy_pom_content),
+ BytesIO(dummy_pom_content),
+ "application/xml",
+ )
+
+ # Add the files to the build
+ removeSecurityProxy(self.build).addFile(jar_lfa)
+ removeSecurityProxy(self.build).addFile(pom_lfa)
+
+ # Create and run the job
+ job = getUtility(ICraftPublishingJobSource).create(self.build)
+ JobRunner([job]).runAll()
+ job = removeSecurityProxy(job)
+
+ self.assertEqual(JobStatus.FAILED, job.job.status)
+ self.assertEqual(
+ "Missing Maven publishing repository configuration",
+ job.error_message,
+ )
diff --git a/lib/lp/crafts/tests/test_craftrecipejob.py b/lib/lp/crafts/tests/test_craftrecipejob.py
index 6ddbe00..11f3b47 100644
--- a/lib/lp/crafts/tests/test_craftrecipejob.py
+++ b/lib/lp/crafts/tests/test_craftrecipejob.py
@@ -3,13 +3,9 @@
"""Tests for craft recipe jobs."""
-import io
-import json
-import tarfile
from textwrap import dedent
import six
-from fixtures import FakeLogger
from testtools.matchers import (
AfterPreprocessing,
ContainsDict,
@@ -22,7 +18,6 @@ from testtools.matchers import (
MatchesStructure,
)
from zope.component import getUtility
-from zope.security.proxy import removeSecurityProxy
from lp.app.interfaces.launchpad import ILaunchpadCelebrities
from lp.code.tests.helpers import GitHostingFixture
@@ -31,13 +26,10 @@ from lp.crafts.interfaces.craftrecipe import (
CannotParseSourcecraftYaml,
)
from lp.crafts.interfaces.craftrecipejob import (
- ICraftPublishingJob,
- ICraftPublishingJobSource,
ICraftRecipeJob,
ICraftRecipeRequestBuildsJob,
)
from lp.crafts.model.craftrecipejob import (
- CraftPublishingJob,
CraftRecipeJob,
CraftRecipeJobType,
CraftRecipeRequestBuildsJob,
@@ -48,11 +40,10 @@ from lp.services.database.sqlbase import get_transaction_timestamp
from lp.services.features.testing import FeatureFixture
from lp.services.job.interfaces.job import JobStatus
from lp.services.job.runner import JobRunner
-from lp.services.librarian.interfaces import ILibraryFileAliasSet
from lp.services.mail.sendmail import format_address_for_person
from lp.testing import TestCaseWithFactory
from lp.testing.dbuser import dbuser
-from lp.testing.layers import CeleryJobLayer, ZopelessDatabaseLayer
+from lp.testing.layers import ZopelessDatabaseLayer
class TestCraftRecipeJob(TestCaseWithFactory):
@@ -274,367 +265,3 @@ class TestCraftRecipeRequestBuildsJob(TestCaseWithFactory):
builds=AfterPreprocessing(set, MatchesSetwise()),
),
)
-
-
-class TestCraftPublishingJob(TestCaseWithFactory):
- """Test the CraftPublishingJob."""
-
- layer = CeleryJobLayer
-
- def setUp(self):
- super().setUp()
- self.useFixture(
- FeatureFixture(
- {"jobs.celery.enabled_classes": "CraftPublishingJob"}
- )
- )
- self.useFixture(FakeLogger())
- self.useFixture(FeatureFixture({CRAFT_RECIPE_ALLOW_CREATE: "on"}))
- self.recipe = self.factory.makeCraftRecipe()
- self.build = self.factory.makeCraftRecipeBuild(recipe=self.recipe)
-
- def test_provides_interface(self):
- # CraftPublishingJob provides ICraftPublishingJob.
- job = getUtility(ICraftPublishingJobSource).create(self.build)
-
- # Check that the instance provides the job interface
- self.assertProvides(job, ICraftPublishingJob)
-
- # Check that the class provides the source interface
- self.assertProvides(CraftPublishingJob, ICraftPublishingJobSource)
-
- def test_create(self):
- # CraftPublishingJob.create creates a CraftPublishingJob with the
- # correct attributes.
- job = getUtility(ICraftPublishingJobSource).create(self.build)
-
- job = removeSecurityProxy(job)
-
- self.assertThat(
- job,
- MatchesStructure(
- class_job_type=Equals(CraftRecipeJobType.PUBLISH_ARTIFACTS),
- recipe=Equals(self.recipe),
- build=Equals(self.build),
- error_message=Is(None),
- ),
- )
-
- def test_run_failure_cannot_determine_distribution(self):
- """Test failure when distribution cannot be determined."""
- job = getUtility(ICraftPublishingJobSource).create(self.build)
-
- # Create a mock git repository with a non-distribution target
- git_repo = self.factory.makeGitRepository()
- self.patch(
- removeSecurityProxy(self.recipe), "git_repository", git_repo
- )
-
- JobRunner([job]).runAll()
-
- job = removeSecurityProxy(job)
- self.assertEqual(JobStatus.FAILED, job.job.status)
- self.assertEqual(
- "Could not determine distribution for build",
- job.error_message,
- )
-
- def test_run_failure_no_configuration(self):
- """Test failure when no configuration is found for the distribution."""
- # Create a distribution with a name that won't have a configuration
- distribution = self.factory.makeDistribution(name="nonexistent")
-
- # Create a distribution source package for our distribution
- package = self.factory.makeDistributionSourcePackage(
- distribution=distribution
- )
-
- # Create a git repository targeting that package
- git_repository = self.factory.makeGitRepository(target=package)
-
- # Update our recipe to use this git repository
- removeSecurityProxy(self.recipe).git_repository = git_repository
-
- job = getUtility(ICraftPublishingJobSource).create(self.build)
- JobRunner([job]).runAll()
- job = removeSecurityProxy(job)
-
- self.assertEqual(JobStatus.FAILED, job.job.status)
- self.assertEqual(
- "No configuration found for nonexistent",
- job.error_message,
- )
-
- def test_run_no_publishable_artifacts(self):
- """Test failure when no publishable artifacts are found."""
- distribution = self.factory.makeDistribution(name="soss")
-
- # Set up config with environment variables but no Cargo publishing info
- # We just need a config section for the distribution name
- self.pushConfig("craftbuild.soss")
- package = self.factory.makeDistributionSourcePackage(
- distribution=distribution
- )
- git_repository = self.factory.makeGitRepository(target=package)
- removeSecurityProxy(self.recipe).git_repository = git_repository
-
- # Create a dummy file (but not a crate or jar)
- from io import BytesIO
-
- dummy_content = b"test content"
-
- # Create a LibraryFileAlias with the dummy content
- librarian = getUtility(ILibraryFileAliasSet)
- lfa = librarian.create(
- "test.txt",
- len(dummy_content),
- BytesIO(dummy_content),
- "text/plain",
- )
-
- # Add the file to the build
- removeSecurityProxy(self.build).addFile(lfa)
-
- job = getUtility(ICraftPublishingJobSource).create(self.build)
- JobRunner([job]).runAll()
- job = removeSecurityProxy(job)
- self.assertEqual(JobStatus.FAILED, job.job.status)
-
- self.assertEqual(
- "No publishable artifacts found in build",
- job.error_message,
- )
-
- def test_run_missing_cargo_config(self):
- """Test failure when a crate is found but Cargo config is missing."""
- distribution = self.factory.makeDistribution(name="soss")
- self.pushConfig("craftbuild.soss")
- package = self.factory.makeDistributionSourcePackage(
- distribution=distribution
- )
-
- git_repository = self.factory.makeGitRepository(target=package)
-
- removeSecurityProxy(self.recipe).git_repository = git_repository
-
- # Create a BytesIO object to hold the tar data
- tar_data = io.BytesIO()
-
- # Create a tar archive
- with tarfile.open(fileobj=tar_data, mode="w") as tar:
- # Create a directory entry for the crate
- crate_dir_info = tarfile.TarInfo("test-0.1.0")
- crate_dir_info.type = tarfile.DIRTYPE
- tar.addfile(crate_dir_info)
-
- # Add a Cargo.toml file
- cargo_toml = """
-[package]
-name = "test"
-version = "0.1.0"
-authors = ["Test <test@xxxxxxxxxxx>"]
-edition = "2018"
-"""
- cargo_toml_info = tarfile.TarInfo("test-0.1.0/Cargo.toml")
- cargo_toml_bytes = cargo_toml.encode("utf-8")
- cargo_toml_info.size = len(cargo_toml_bytes)
- tar.addfile(cargo_toml_info, io.BytesIO(cargo_toml_bytes))
-
- # Add a src directory
- src_dir_info = tarfile.TarInfo("test-0.1.0/src")
- src_dir_info.type = tarfile.DIRTYPE
- tar.addfile(src_dir_info)
-
- # Add a main.rs file
- main_rs = 'fn main() { println!("Hello, world!"); }'
- main_rs_info = tarfile.TarInfo("test-0.1.0/src/main.rs")
- main_rs_bytes = main_rs.encode("utf-8")
- main_rs_info.size = len(main_rs_bytes)
- tar.addfile(main_rs_info, io.BytesIO(main_rs_bytes))
-
- # Get the tar data
- tar_data.seek(0)
- crate_content = tar_data.getvalue()
-
- # Create a LibraryFileAlias with the crate content
- librarian = getUtility(ILibraryFileAliasSet)
- lfa = librarian.create(
- "test-0.1.0.crate",
- len(crate_content),
- io.BytesIO(crate_content),
- "application/x-tar",
- )
-
- removeSecurityProxy(self.build).addFile(lfa)
-
- job = getUtility(ICraftPublishingJobSource).create(self.build)
- JobRunner([job]).runAll()
- job = removeSecurityProxy(job)
-
- self.assertEqual(JobStatus.FAILED, job.job.status)
- self.assertEqual(
- "Missing Cargo publishing repository configuration",
- job.error_message,
- )
-
- def test_run_crate_extraction_failure(self):
- """Test failure when crate extraction fails."""
- distribution = self.factory.makeDistribution(name="soss")
-
- # Set up config with environment variables
- self.pushConfig(
- "craftbuild.soss",
- environment_variables=json.dumps(
- {
- "CARGO_PUBLISH_URL": "https://example.com/registry";,
- "CARGO_PUBLISH_AUTH": "lp:token123",
- }
- ),
- )
-
- package = self.factory.makeDistributionSourcePackage(
- distribution=distribution
- )
-
- git_repository = self.factory.makeGitRepository(target=package)
-
- removeSecurityProxy(self.recipe).git_repository = git_repository
-
- # Create an invalid tar file (just some random bytes)
- invalid_crate_content = b"This is not a valid tar file"
-
- librarian = getUtility(ILibraryFileAliasSet)
- lfa = librarian.create(
- "invalid-0.1.0.crate",
- len(invalid_crate_content),
- io.BytesIO(invalid_crate_content),
- "application/x-tar",
- )
-
- removeSecurityProxy(self.build).addFile(lfa)
-
- job = getUtility(ICraftPublishingJobSource).create(self.build)
- JobRunner([job]).runAll()
- job = removeSecurityProxy(job)
-
- self.assertEqual(JobStatus.FAILED, job.job.status)
- self.assertIn(
- "Failed to extract crate",
- job.error_message,
- )
-
- def test_run_no_directory_in_crate(self):
- """Test failure when no directory is found in extracted crate."""
- distribution = self.factory.makeDistribution(name="soss")
-
- # Set up config with environment variables
- self.pushConfig(
- "craftbuild.soss",
- environment_variables=json.dumps(
- {
- "CARGO_PUBLISH_URL": "https://example.com/registry";,
- "CARGO_PUBLISH_AUTH": "lp:token123",
- }
- ),
- )
-
- package = self.factory.makeDistributionSourcePackage(
- distribution=distribution
- )
-
- git_repository = self.factory.makeGitRepository(target=package)
-
- removeSecurityProxy(self.recipe).git_repository = git_repository
-
- tar_data = io.BytesIO()
-
- # Create a tar archive with only files (no directories)
- with tarfile.open(fileobj=tar_data, mode="w") as tar:
- # Add a file at the root level (no directory)
- file_info = tarfile.TarInfo("file.txt")
- file_content = b"This is a file with no directory"
- file_info.size = len(file_content)
- tar.addfile(file_info, io.BytesIO(file_content))
-
- # Get the tar data
- tar_data.seek(0)
- crate_content = tar_data.getvalue()
-
- # Create a LibraryFileAlias with the crate content
- librarian = getUtility(ILibraryFileAliasSet)
- lfa = librarian.create(
- "nodirs-0.1.0.crate",
- len(crate_content),
- io.BytesIO(crate_content),
- "application/x-tar",
- )
-
- # Add the file to the build
- removeSecurityProxy(self.build).addFile(lfa)
-
- # Create and run the job
- job = getUtility(ICraftPublishingJobSource).create(self.build)
-
- JobRunner([job]).runAll()
-
- # Verify job failed with expected error message
- job = removeSecurityProxy(job)
- self.assertEqual(JobStatus.FAILED, job.job.status)
- self.assertEqual(
- "No directory found in extracted crate",
- job.error_message,
- )
-
- def test_run_missing_maven_config(self):
- """
- Test failure when Maven artifacts are found but Maven config is
- missing.
- """
- distribution = self.factory.makeDistribution(name="soss")
-
- self.pushConfig("craftbuild.soss")
- package = self.factory.makeDistributionSourcePackage(
- distribution=distribution
- )
- git_repository = self.factory.makeGitRepository(target=package)
- removeSecurityProxy(self.recipe).git_repository = git_repository
-
- # Create a dummy jar file
- from io import BytesIO
-
- dummy_jar_content = b"dummy jar content"
-
- # Create a LibraryFileAlias with the jar content
- librarian = getUtility(ILibraryFileAliasSet)
- jar_lfa = librarian.create(
- "test-0.1.0.jar",
- len(dummy_jar_content),
- BytesIO(dummy_jar_content),
- "application/java-archive",
- )
-
- # Create a dummy pom file
- dummy_pom_content = b"<project>...</project>"
-
- # Create a LibraryFileAlias with the pom content
- pom_lfa = librarian.create(
- "pom.xml",
- len(dummy_pom_content),
- BytesIO(dummy_pom_content),
- "application/xml",
- )
-
- # Add the files to the build
- removeSecurityProxy(self.build).addFile(jar_lfa)
- removeSecurityProxy(self.build).addFile(pom_lfa)
-
- # Create and run the job
- job = getUtility(ICraftPublishingJobSource).create(self.build)
- JobRunner([job]).runAll()
- job = removeSecurityProxy(job)
-
- self.assertEqual(JobStatus.FAILED, job.job.status)
- self.assertEqual(
- "Missing Maven publishing repository configuration",
- job.error_message,
- )
