launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #32373
[Merge] ~ruinedyourlife/launchpad:native-tools-publishing into launchpad:master
Quentin Debhi has proposed merging ~ruinedyourlife/launchpad:native-tools-publishing into launchpad:master.
Commit message:
Publish maven and cargo artifacts with native tooling for craft builds
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~ruinedyourlife/launchpad/+git/launchpad/+merge/483691
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~ruinedyourlife/launchpad:native-tools-publishing into launchpad:master.
diff --git a/lib/lp/crafts/configure.zcml b/lib/lp/crafts/configure.zcml
index a13bd3f..59d8621 100644
--- a/lib/lp/crafts/configure.zcml
+++ b/lib/lp/crafts/configure.zcml
@@ -63,6 +63,11 @@
interface="lp.crafts.interfaces.craftrecipebuild.ICraftRecipeBuildAdmin" />
</class>
+ <subscriber
+ for="lp.crafts.interfaces.craftrecipebuild.ICraftRecipeBuild
+ lp.crafts.interfaces.craftrecipebuild.ICraftRecipeBuildStatusChangedEvent"
+ handler="lp.crafts.subscribers.craftrecipebuild.craft_build_status_changed" />
+
<!-- CraftRecipeBuildSet -->
<lp:securedutility
class="lp.crafts.model.craftrecipebuild.CraftRecipeBuildSet"
@@ -110,4 +115,15 @@
name="craft-recipe-build">
<allow interface="lp.services.macaroons.interfaces.IMacaroonIssuerPublic"/>
</lp:securedutility>
+
+ <!-- CraftPublishingJob -->
+ <lp:securedutility
+ component="lp.crafts.model.craftrecipejob.CraftPublishingJob"
+ provides="lp.crafts.interfaces.craftrecipejob.ICraftPublishingJobSource">
+ <allow interface="lp.crafts.interfaces.craftrecipejob.ICraftPublishingJobSource" />
+ </lp:securedutility>
+ <class class="lp.crafts.model.craftrecipejob.CraftPublishingJob">
+ <allow interface="lp.crafts.interfaces.craftrecipejob.ICraftRecipeJob" />
+ <allow interface="lp.crafts.interfaces.craftrecipejob.ICraftPublishingJob" />
+ </class>
</configure>
diff --git a/lib/lp/crafts/interfaces/craftrecipebuild.py b/lib/lp/crafts/interfaces/craftrecipebuild.py
index 7b57a27..60e2a54 100644
--- a/lib/lp/crafts/interfaces/craftrecipebuild.py
+++ b/lib/lp/crafts/interfaces/craftrecipebuild.py
@@ -7,6 +7,7 @@ __all__ = [
"ICraftFile",
"ICraftRecipeBuild",
"ICraftRecipeBuildSet",
+ "ICraftRecipeBuildStatusChangedEvent",
]
from lazr.restful.declarations import (
@@ -17,6 +18,7 @@ from lazr.restful.declarations import (
)
from lazr.restful.fields import Reference
from zope.interface import Attribute, Interface
+from zope.interface.interfaces import IObjectEvent
from zope.schema import Bool, Datetime, Dict, Int, TextLine
from lp import _
@@ -38,6 +40,10 @@ from lp.services.librarian.interfaces import ILibraryFileAlias
from lp.soyuz.interfaces.distroarchseries import IDistroArchSeries
+class ICraftRecipeBuildStatusChangedEvent(IObjectEvent):
+ """The status of a craft recipe build changed."""
+
+
class ICraftRecipeBuildView(IPackageBuildView):
"""ICraftRecipeBuild attributes that require launchpad.View."""
diff --git a/lib/lp/crafts/interfaces/craftrecipejob.py b/lib/lp/crafts/interfaces/craftrecipejob.py
index addc6ee..81ea511 100644
--- a/lib/lp/crafts/interfaces/craftrecipejob.py
+++ b/lib/lp/crafts/interfaces/craftrecipejob.py
@@ -7,6 +7,8 @@ __all__ = [
"ICraftRecipeJob",
"ICraftRecipeRequestBuildsJob",
"ICraftRecipeRequestBuildsJobSource",
+ "ICraftPublishingJob",
+ "ICraftPublishingJobSource",
]
from lazr.restful.fields import Reference
@@ -136,3 +138,34 @@ class ICraftRecipeRequestBuildsJobSource(IJobSource):
recipe and ID, or its `job_type` is not
`CraftRecipeJobType.REQUEST_BUILDS`.
"""
+
+
+class ICraftPublishingJob(IRunnableJob):
+ """
+ A job that publishes craft recipe build artifacts to external repositories.
+ """
+
+ build_id = Attribute("The ID of the build to publish.")
+ build = Attribute("The build to publish.")
+ error_message = Attribute("The error message if the publishing failed.")
+
+ def create(build):
+ """Create a new CraftPublishingJob."""
+
+ def getPublishingType():
+ """Return the publishing type that this job will handle.
+
+ Each type corresponds to a specific artifact type and publishing
+ destination.
+ """
+
+
+class ICraftPublishingJobSource(IJobSource):
+ """A source for creating and finding CraftPublishingJobs."""
+
+ def create(build):
+ """
+ Publish artifacts from a craft recipe build to external repositories.
+
+ :param build: The build to publish.
+ """
diff --git a/lib/lp/crafts/model/craftrecipebuild.py b/lib/lp/crafts/model/craftrecipebuild.py
index 8fa4baa..1ecd39c 100644
--- a/lib/lp/crafts/model/craftrecipebuild.py
+++ b/lib/lp/crafts/model/craftrecipebuild.py
@@ -6,6 +6,7 @@
__all__ = [
"CraftFile",
"CraftRecipeBuild",
+ "ICraftRecipeBuildStatusChangedEvent",
]
from datetime import timedelta, timezone
@@ -16,6 +17,7 @@ from storm.locals import Bool, DateTime, Desc, Int, Reference, Store, Unicode
from storm.store import EmptyResultSet
from zope.component import getUtility
from zope.interface import implementer
+from zope.interface.interfaces import ObjectEvent
from zope.security.proxy import removeSecurityProxy
from lp.app.errors import NotFoundError
@@ -33,6 +35,7 @@ from lp.crafts.interfaces.craftrecipebuild import (
ICraftFile,
ICraftRecipeBuild,
ICraftRecipeBuildSet,
+ ICraftRecipeBuildStatusChangedEvent,
)
from lp.crafts.mail.craftrecipebuild import CraftRecipeBuildMailer
from lp.registry.interfaces.pocket import PackagePublishingPocket
@@ -60,6 +63,11 @@ from lp.services.webapp.snapshot import notify_modified
from lp.soyuz.model.distroarchseries import DistroArchSeries
+@implementer(ICraftRecipeBuildStatusChangedEvent)
+class CraftRecipeBuildStatusChangedEvent(ObjectEvent):
+ """See `ICraftRecipeBuildStatusChangedEvent`."""
+
+
@implementer(ICraftRecipeBuild)
class CraftRecipeBuild(PackageBuildMixin, StormBase):
"""See `ICraftRecipeBuild`."""
diff --git a/lib/lp/crafts/model/craftrecipejob.py b/lib/lp/crafts/model/craftrecipejob.py
index d573665..1419fe9 100644
--- a/lib/lp/crafts/model/craftrecipejob.py
+++ b/lib/lp/crafts/model/craftrecipejob.py
@@ -7,8 +7,18 @@ __all__ = [
"CraftRecipeJob",
"CraftRecipeJobType",
"CraftRecipeRequestBuildsJob",
+ "CraftPublishingJob",
]
+import json
+import lzma
+import os
+import subprocess
+import tarfile
+import tempfile
+from configparser import NoSectionError
+from pathlib import Path
+
import transaction
from lazr.delegates import delegate_to
from lazr.enum import DBEnumeratedType, DBItem
@@ -25,11 +35,16 @@ from lp.crafts.interfaces.craftrecipe import (
MissingSourcecraftYaml,
)
from lp.crafts.interfaces.craftrecipejob import (
+ ICraftPublishingJob,
+ ICraftPublishingJobSource,
ICraftRecipeJob,
ICraftRecipeRequestBuildsJob,
ICraftRecipeRequestBuildsJobSource,
)
from lp.crafts.model.craftrecipebuild import CraftRecipeBuild
+from lp.registry.interfaces.distributionsourcepackage import (
+ IDistributionSourcePackage,
+)
from lp.registry.interfaces.person import IPersonSet
from lp.services.config import config
from lp.services.database.bulk import load_related
@@ -56,6 +71,16 @@ class CraftRecipeJobType(DBEnumeratedType):
""",
)
+ PUBLISH_ARTIFACTS = DBItem(
+ 1,
+ """
+ Publish artifacts
+
+ This job publishes craft recipe build artifacts to external
+ repositories.
+ """,
+ )
+
@implementer(ICraftRecipeJob)
class CraftRecipeJob(StormBase):
@@ -337,3 +362,331 @@ class CraftRecipeRequestBuildsJob(CraftRecipeJobDerived):
# are to this job's metadata and should be preserved.
transaction.commit()
raise
+
+
+@implementer(ICraftPublishingJob)
+@provider(ICraftPublishingJobSource)
+class CraftPublishingJob(CraftRecipeJobDerived):
+ """
+ A Job that publishes craft recipe build artifacts to external
+ repositories.
+ """
+
+ class_job_type = CraftRecipeJobType.PUBLISH_ARTIFACTS
+
+ user_error_types = ()
+ retry_error_types = ()
+ max_retries = 5
+
+ config = config.ICraftPublishingJobSource
+
+ @classmethod
+ def create(cls, build):
+ """See `ICraftPublishingJobSource`."""
+ cls.metadata = {
+ "build_id": build.id,
+ }
+ recipe_job = CraftRecipeJob(
+ build.recipe, cls.class_job_type, cls.metadata
+ )
+ job = cls(recipe_job)
+ job.celeryRunOnCommit()
+ IStore(CraftRecipeJob).flush()
+ return job
+
+ @property
+ def build_id(self):
+ """See `ICraftPublishingJob`."""
+ return self.metadata["build_id"]
+
+ @cachedproperty
+ def build(self):
+ """See `ICraftPublishingJob`."""
+ return IStore(CraftRecipeBuild).get(CraftRecipeBuild, self.build_id)
+
+ @property
+ def error_message(self):
+ """See `ICraftPublishingJob`."""
+ return self.metadata.get("error_message")
+
+ @error_message.setter
+ def error_message(self, message):
+ """See `ICraftPublishingJob`."""
+ self.metadata["error_message"] = message
+
+ def getPublishingType(self, archive_path):
+ """Determine the type of artifacts to publish.
+
+ :param archive_path: Optional path to an already downloaded archive
+ file
+ :return: "cargo", "maven", or None if no publishable artifacts found
+ """
+ try:
+ has_crate = False
+ has_jar = False
+ has_pom = False
+
+ with lzma.open(archive_path) as xz:
+ with tarfile.open(fileobj=xz) as tar:
+ for member in tar.getmembers():
+ if member.name.endswith(".crate"):
+ has_crate = True
+ elif member.name.endswith(".jar"):
+ has_jar = True
+ elif member.name == "pom.xml":
+ has_pom = True
+
+ if has_crate:
+ return "cargo"
+ if has_jar and has_pom:
+ return "maven"
+ except Exception:
+ pass
+
+ return None
+
+ def run(self):
+ """See `IRunnableJob`."""
+ try:
+ # Get the distribution name to access the correct configuration
+ distribution_name = None
+ git_repo = self.build.recipe.git_repository
+ if git_repo is not None:
+ if IDistributionSourcePackage.providedBy(git_repo.target):
+ distribution_name = git_repo.target.distribution.name
+
+ if not distribution_name:
+ self.error_message = (
+ "Could not determine distribution for build"
+ )
+ raise Exception(self.error_message)
+
+ # Get environment variables from configuration
+ try:
+ env_vars_json = config["craftbuild." + distribution_name][
+ "environment_variables"
+ ]
+ if env_vars_json and env_vars_json.lower() != "none":
+ env_vars = json.loads(env_vars_json)
+ # Replace auth placeholders
+ for key, value in env_vars.items():
+ if (
+ isinstance(value, str)
+ and "%(write_auth)s" in value
+ ):
+ env_vars[key] = value.replace(
+ "%(write_auth)s",
+ config.artifactory.write_credentials,
+ )
+ else:
+ env_vars = {}
+ except (NoSectionError, KeyError):
+ self.error_message = (
+ f"No configuration found for {distribution_name}"
+ )
+ raise Exception(self.error_message)
+
+ # Find the archive file in the build
+ archive_file = None
+ for _, lfa, _ in self.build.getFiles():
+ if lfa.filename.endswith(".tar.xz"):
+ archive_file = lfa
+ break
+
+ if archive_file is None:
+ raise Exception("No archive file found in build")
+
+ # Process the archive file
+ with tempfile.TemporaryDirectory() as tmpdir:
+ # Download the archive file to a temporary location
+ archive_path = os.path.join(tmpdir, "archive.tar.xz")
+
+ # Use the correct pattern for LibraryFileAlias
+ archive_file.open() # This sets _datafile but returns None
+ try:
+ with open(archive_path, "wb") as f:
+ f.write(archive_file.read())
+ finally:
+ archive_file.close()
+
+ # Determine what to publish using the getPublishingType method
+ publishing_type = self.getPublishingType(archive_path)
+
+ if publishing_type is None:
+ raise Exception(
+ "No publishable artifacts found in archive"
+ )
+
+ # Extract the archive
+ extract_dir = os.path.join(tmpdir, "extract")
+ os.makedirs(extract_dir, exist_ok=True)
+
+ try:
+ with lzma.open(archive_path) as xz:
+ with tarfile.open(fileobj=xz, mode="r") as tar:
+ tar.extractall(path=extract_dir)
+ except Exception as e:
+ raise Exception(f"Failed to extract archive: {str(e)}")
+
+ # Publish artifacts based on type
+ if publishing_type == "cargo":
+ self._publish_rust_crate(extract_dir, env_vars)
+ elif publishing_type == "maven":
+ self._publish_maven_artifact(extract_dir, env_vars)
+
+ except Exception as e:
+ self.error_message = str(e)
+ # The normal job infrastructure will abort the transaction, but
+ # we want to commit instead: the only database changes we make
+ # are to this job's metadata and should be preserved.
+ transaction.commit()
+ raise
+
+ def _publish_rust_crate(self, extract_dir, env_vars):
+ """Publish Rust crates found in the extracted archive.
+
+ :param extract_dir: Path to the extracted archive
+ :param env_vars: Environment variables from configuration
+ :raises: Exception if publishing fails
+ """
+ # Look for specific Cargo publishing repository configuration
+ cargo_publish_url = env_vars.get("CARGO_PUBLISH_URL")
+ cargo_publish_auth = env_vars.get("CARGO_PUBLISH_AUTH")
+
+ if not cargo_publish_url or not cargo_publish_auth:
+ raise Exception(
+ "Missing Cargo publishing repository configuration"
+ )
+
+ # Look for .crate files
+ crate_files = list(Path(extract_dir).rglob("*.crate"))
+ crate_file = crate_files[0]
+
+ # Set up cargo config
+ cargo_dir = os.path.join(extract_dir, ".cargo")
+ os.makedirs(cargo_dir, exist_ok=True)
+
+ # Create config.toml
+ with open(os.path.join(cargo_dir, "config.toml"), "w") as f:
+ f.write(
+ """
+[registry]
+global-credential-providers = ["cargo:token"]
+
+[registries.launchpad]
+index = "{}"
+""".format(
+ cargo_publish_url
+ )
+ )
+
+ # Create credentials.toml
+ with open(os.path.join(cargo_dir, "credentials.toml"), "w") as f:
+ f.write(
+ """
+[registries.launchpad]
+token = "Bearer {}"
+""".format(
+ cargo_publish_auth
+ )
+ )
+
+ # Run cargo publish
+ result = subprocess.run(
+ [
+ "cargo",
+ "publish",
+ "--no-verify",
+ "--allow-dirty",
+ "--registry",
+ "launchpad",
+ ],
+ cwd=os.path.dirname(str(crate_file)),
+ env={"CARGO_HOME": cargo_dir},
+ capture_output=True,
+ text=True,
+ )
+
+ if result.returncode != 0:
+ raise Exception(f"Failed to publish crate: {result.stderr}")
+
+ def _publish_maven_artifact(self, extract_dir, env_vars):
+ """Publish Maven artifacts found in the extracted archive.
+
+ :param extract_dir: Path to the extracted archive
+ :param env_vars: Environment variables from configuration
+ :raises: Exception if publishing fails
+ """
+ # Look for specific Maven publishing repository configuration
+ maven_publish_url = env_vars.get("MAVEN_PUBLISH_URL")
+ maven_publish_auth = env_vars.get("MAVEN_PUBLISH_AUTH")
+
+ if not maven_publish_url or not maven_publish_auth:
+ raise Exception(
+ "Missing Maven publishing repository configuration"
+ )
+
+ # Find the JAR file and pom.xml in the extracted archive
+ jar_file = None
+ pom_file = None
+
+ for root, _, files in os.walk(extract_dir):
+ for filename in files:
+ if filename.endswith(".jar"):
+ jar_file = os.path.join(root, filename)
+ elif filename == "pom.xml":
+ pom_file = os.path.join(root, filename)
+
+ if jar_file is None:
+ raise Exception("No .jar file found in archive")
+
+ if pom_file is None:
+ raise Exception("No pom.xml file found in archive")
+
+ # Set up Maven settings
+ maven_dir = os.path.join(extract_dir, ".m2")
+ os.makedirs(maven_dir, exist_ok=True)
+
+ # Extract username and password from auth string
+ if ":" in maven_publish_auth:
+ username, password = maven_publish_auth.split(":", 1)
+ else:
+ username = "token"
+ password = maven_publish_auth
+
+ # Create settings.xml with server configuration for the
+ # publishing repository
+ settings_xml = f"""<settings>
+<servers>
+ <server>
+ <id>launchpad-publish</id>
+ <username>{username}</username>
+ <password>{password}</password>
+ </server>
+</servers>
+</settings>"""
+
+ with open(os.path.join(maven_dir, "settings.xml"), "w") as f:
+ f.write(settings_xml)
+
+ # Run mvn deploy using the pom file
+ result = subprocess.run(
+ [
+ "mvn",
+ "deploy:deploy-file",
+ f"-DpomFile={pom_file}",
+ f"-Dfile={jar_file}",
+ "-DrepositoryId=launchpad-publish",
+ f"-Durl={maven_publish_url}",
+ "--settings={}".format(
+ os.path.join(maven_dir, "settings.xml")
+ ),
+ ],
+ capture_output=True,
+ text=True,
+ )
+
+ if result.returncode != 0:
+ raise Exception(
+ f"Failed to publish Maven artifact: {result.stderr}"
+ )
diff --git a/lib/lp/crafts/subscribers/__init__.py b/lib/lp/crafts/subscribers/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/lib/lp/crafts/subscribers/__init__.py
diff --git a/lib/lp/crafts/subscribers/craftrecipebuild.py b/lib/lp/crafts/subscribers/craftrecipebuild.py
new file mode 100644
index 0000000..5b29d9d
--- /dev/null
+++ b/lib/lp/crafts/subscribers/craftrecipebuild.py
@@ -0,0 +1,71 @@
+# lib/lp/crafts/subscribers/craftrecipebuild.py
+
+"""Event subscribers for craft recipe builds."""
+
+from configparser import NoSectionError
+
+from zope.component import getUtility
+
+from lp.buildmaster.enums import BuildStatus
+from lp.crafts.interfaces.craftrecipebuild import ICraftRecipeBuild
+from lp.crafts.interfaces.craftrecipejob import ICraftPublishingJobSource
+from lp.registry.interfaces.distributionsourcepackage import (
+ IDistributionSourcePackage,
+)
+from lp.services.config import config
+from lp.services.scripts import log
+from lp.services.webapp.publisher import canonical_url
+from lp.services.webhooks.interfaces import IWebhookSet
+from lp.services.webhooks.payload import compose_webhook_payload
+
+
+def _trigger_craft_build_webhook(build, action):
+ """Trigger a webhook for a craft recipe build event."""
+ payload = {
+ "craft_build": canonical_url(build, force_local_path=True),
+ "action": action,
+ }
+ payload.update(
+ compose_webhook_payload(
+ ICraftRecipeBuild,
+ build,
+ ["recipe", "build_request", "status"],
+ )
+ )
+ getUtility(IWebhookSet).trigger(
+ build.recipe, "craft-recipe:build:0.1", payload
+ )
+
+
+def craft_build_status_changed(build, event):
+ """Trigger events when craft recipe build statuses change."""
+ _trigger_craft_build_webhook(build, "status-changed")
+
+ if build.status == BuildStatus.FULLYBUILT:
+ # Check if this build is from a configured distribution
+ should_publish = False
+ if (
+ build.recipe.git_repository is not None
+ and IDistributionSourcePackage.providedBy(
+ build.recipe.git_repository.target
+ )
+ ):
+ distribution_name = (
+ build.recipe.git_repository.target.distribution.name
+ )
+ try:
+ # Check if there are any config variables for this distribution
+ config["craftbuild." + distribution_name]
+ should_publish = True
+ except NoSectionError:
+ # If no section is found, we shouldn't publish
+ should_publish = False
+ log.debug(
+ "No configuration found for distribution %s, "
+ "skipping upload" % distribution_name
+ )
+
+ # Only schedule uploads for configured distribution builds
+ if should_publish and build.recipe.store_upload:
+ log.info("Scheduling publishing of artifacts from %r" % build)
+ getUtility(ICraftPublishingJobSource).create(build)
diff --git a/lib/lp/crafts/tests/test_craftrecipejob.py b/lib/lp/crafts/tests/test_craftrecipejob.py
index 11f3b47..1809f3c 100644
--- a/lib/lp/crafts/tests/test_craftrecipejob.py
+++ b/lib/lp/crafts/tests/test_craftrecipejob.py
@@ -6,6 +6,7 @@
from textwrap import dedent
import six
+from fixtures import FakeLogger
from testtools.matchers import (
AfterPreprocessing,
ContainsDict,
@@ -18,6 +19,7 @@ from testtools.matchers import (
MatchesStructure,
)
from zope.component import getUtility
+from zope.security.proxy import removeSecurityProxy
from lp.app.interfaces.launchpad import ILaunchpadCelebrities
from lp.code.tests.helpers import GitHostingFixture
@@ -26,10 +28,13 @@ from lp.crafts.interfaces.craftrecipe import (
CannotParseSourcecraftYaml,
)
from lp.crafts.interfaces.craftrecipejob import (
+ ICraftPublishingJob,
+ ICraftPublishingJobSource,
ICraftRecipeJob,
ICraftRecipeRequestBuildsJob,
)
from lp.crafts.model.craftrecipejob import (
+ CraftPublishingJob,
CraftRecipeJob,
CraftRecipeJobType,
CraftRecipeRequestBuildsJob,
@@ -40,10 +45,11 @@ from lp.services.database.sqlbase import get_transaction_timestamp
from lp.services.features.testing import FeatureFixture
from lp.services.job.interfaces.job import JobStatus
from lp.services.job.runner import JobRunner
+from lp.services.librarian.interfaces import ILibraryFileAliasSet
from lp.services.mail.sendmail import format_address_for_person
from lp.testing import TestCaseWithFactory
from lp.testing.dbuser import dbuser
-from lp.testing.layers import ZopelessDatabaseLayer
+from lp.testing.layers import CeleryJobLayer, ZopelessDatabaseLayer
class TestCraftRecipeJob(TestCaseWithFactory):
@@ -265,3 +271,293 @@ class TestCraftRecipeRequestBuildsJob(TestCaseWithFactory):
builds=AfterPreprocessing(set, MatchesSetwise()),
),
)
+
+
+class TestCraftPublishingJob(TestCaseWithFactory):
+ """Test the CraftPublishingJob."""
+
+ layer = CeleryJobLayer
+
+ def setUp(self):
+ super().setUp()
+ self.useFixture(
+ FeatureFixture(
+ {"jobs.celery.enabled_classes": "CraftPublishingJob"}
+ )
+ )
+ self.useFixture(FakeLogger())
+ self.useFixture(FeatureFixture({CRAFT_RECIPE_ALLOW_CREATE: "on"}))
+ self.recipe = self.factory.makeCraftRecipe()
+ self.build = self.factory.makeCraftRecipeBuild(recipe=self.recipe)
+
+ def test_provides_interface(self):
+ # CraftPublishingJob provides ICraftPublishingJob.
+ job = getUtility(ICraftPublishingJobSource).create(self.build)
+
+ # Check that the instance provides the job interface
+ self.assertProvides(job, ICraftPublishingJob)
+
+ # Check that the class provides the source interface
+ self.assertProvides(CraftPublishingJob, ICraftPublishingJobSource)
+
+ def test_create(self):
+ # CraftPublishingJob.create creates a CraftPublishingJob with the
+ # correct attributes.
+ job = getUtility(ICraftPublishingJobSource).create(self.build)
+
+ job = removeSecurityProxy(job)
+
+ self.assertThat(
+ job,
+ MatchesStructure(
+ class_job_type=Equals(CraftRecipeJobType.PUBLISH_ARTIFACTS),
+ recipe=Equals(self.recipe),
+ build=Equals(self.build),
+ error_message=Is(None),
+ ),
+ )
+
+ def test_run_failure_cannot_determine_distribution(self):
+ """Test failure when distribution cannot be determined."""
+ job = getUtility(ICraftPublishingJobSource).create(self.build)
+
+ # Create a mock git repository with a non-distribution target
+ git_repo = self.factory.makeGitRepository()
+ self.patch(
+ removeSecurityProxy(self.recipe), "git_repository", git_repo
+ )
+
+ JobRunner([job]).runAll()
+
+ job = removeSecurityProxy(job)
+ self.assertEqual(JobStatus.FAILED, job.job.status)
+ self.assertEqual(
+ "Could not determine distribution for build",
+ job.error_message,
+ )
+
+ def test_run_failure_no_configuration(self):
+ """Test failure when no configuration is found for the distribution."""
+ # Create a distribution with a name that won't have a configuration
+ distribution = self.factory.makeDistribution(name="nonexistent")
+
+ # Create a distribution source package for our distribution
+ package = self.factory.makeDistributionSourcePackage(
+ distribution=distribution
+ )
+
+ # Create a git repository targeting that package
+ git_repository = self.factory.makeGitRepository(target=package)
+
+ # Update our recipe to use this git repository
+ removeSecurityProxy(self.recipe).git_repository = git_repository
+
+ job = getUtility(ICraftPublishingJobSource).create(self.build)
+ JobRunner([job]).runAll()
+ job = removeSecurityProxy(job)
+
+ self.assertEqual(JobStatus.FAILED, job.job.status)
+ self.assertEqual(
+ "No configuration found for nonexistent",
+ job.error_message,
+ )
+
+ def test_run_failure_no_archive_file(self):
+ """Test failure when no archive file is found in the build."""
+ distribution = self.factory.makeDistribution(name="soss")
+
+ # Set up config with environment variables but no Cargo publishing info
+ # We just need a config section for the distribution name
+ self.pushConfig("craftbuild.soss")
+ package = self.factory.makeDistributionSourcePackage(
+ distribution=distribution
+ )
+ git_repository = self.factory.makeGitRepository(target=package)
+ removeSecurityProxy(self.recipe).git_repository = git_repository
+
+ # Create a job without adding any files to the build
+ job = getUtility(ICraftPublishingJobSource).create(self.build)
+
+ # Run the job - it should fail because there are no files
+ JobRunner([job]).runAll()
+
+ # Verify job failed with error message
+ job = removeSecurityProxy(job)
+
+ self.assertEqual(JobStatus.FAILED, job.job.status)
+ self.assertEqual(
+ "No archive file found in build",
+ job.error_message,
+ )
+
+ def test_run_no_publishable_artifacts(self):
+ """Test failure when no publishable artifacts are found."""
+ distribution = self.factory.makeDistribution(name="soss")
+
+ # Set up config with environment variables but no Cargo publishing info
+ # We just need a config section for the distribution name
+ self.pushConfig("craftbuild.soss")
+ package = self.factory.makeDistributionSourcePackage(
+ distribution=distribution
+ )
+ git_repository = self.factory.makeGitRepository(target=package)
+ removeSecurityProxy(self.recipe).git_repository = git_repository
+
+ # Create a tar.xz file with a dummy file (but no artifacts)
+ import lzma
+ import tarfile
+ from io import BytesIO
+
+ # Create a tar file in memory
+ tar_content = BytesIO()
+ with tarfile.open(fileobj=tar_content, mode="w") as tar:
+ # Add a dummy text file
+ info = tarfile.TarInfo("test.txt")
+ dummy_content = b"test content"
+ info.size = len(dummy_content)
+ tar.addfile(info, BytesIO(dummy_content))
+
+ # Compress the tar content with lzma
+ tar_content.seek(0)
+ xz_content = BytesIO()
+ with lzma.open(xz_content, "w") as xz:
+ xz.write(tar_content.read())
+
+ xz_content.seek(0)
+ tar_xz_content = xz_content.read()
+
+ # Create a LibraryFileAlias with the tar.xz content
+ librarian = getUtility(ILibraryFileAliasSet)
+ lfa = librarian.create(
+ "archive.tar.xz",
+ len(tar_xz_content),
+ BytesIO(tar_xz_content),
+ "application/x-xz",
+ )
+
+ # Add the file to the build
+ removeSecurityProxy(self.build).addFile(lfa)
+
+ job = getUtility(ICraftPublishingJobSource).create(self.build)
+ JobRunner([job]).runAll()
+ job = removeSecurityProxy(job)
+
+ self.assertEqual(JobStatus.FAILED, job.job.status)
+ self.assertEqual(
+ "No publishable artifacts found in archive",
+ job.error_message,
+ )
+
+ def test_run_missing_cargo_config(self):
+ """Test failure when a crate is found but Cargo config is missing."""
+ distribution = self.factory.makeDistribution(name="soss")
+ self.pushConfig("craftbuild.soss")
+ package = self.factory.makeDistributionSourcePackage(
+ distribution=distribution
+ )
+
+ git_repository = self.factory.makeGitRepository(target=package)
+
+ removeSecurityProxy(self.recipe).git_repository = git_repository
+
+ # Create a tar.xz file with a .crate file
+ import lzma
+ import tarfile
+ from io import BytesIO
+
+ tar_content = BytesIO()
+ with tarfile.open(fileobj=tar_content, mode="w") as tar:
+ # Add a dummy crate file
+ info = tarfile.TarInfo("test-0.1.0.crate")
+ dummy_content = b"dummy crate content"
+ info.size = len(dummy_content)
+ tar.addfile(info, BytesIO(dummy_content))
+
+ tar_content.seek(0)
+ xz_content = BytesIO()
+ with lzma.open(xz_content, "w") as xz:
+ xz.write(tar_content.read())
+
+ xz_content.seek(0)
+ tar_xz_content = xz_content.read()
+
+ librarian = getUtility(ILibraryFileAliasSet)
+ lfa = librarian.create(
+ "archive.tar.xz",
+ len(tar_xz_content),
+ BytesIO(tar_xz_content),
+ "application/x-xz",
+ )
+
+ removeSecurityProxy(self.build).addFile(lfa)
+
+ job = getUtility(ICraftPublishingJobSource).create(self.build)
+ JobRunner([job]).runAll()
+ job = removeSecurityProxy(job)
+
+ self.assertEqual(JobStatus.FAILED, job.job.status)
+ self.assertEqual(
+ "Missing Cargo publishing repository configuration",
+ job.error_message,
+ )
+
+ def test_run_missing_maven_config(self):
+ """
+ Test failure when Maven artifacts are found but Maven config is
+ missing.
+ """
+ distribution = self.factory.makeDistribution(name="soss")
+
+ self.pushConfig("craftbuild.soss")
+ package = self.factory.makeDistributionSourcePackage(
+ distribution=distribution
+ )
+ git_repository = self.factory.makeGitRepository(target=package)
+ removeSecurityProxy(self.recipe).git_repository = git_repository
+
+ # Create a tar.xz file with a .jar file and pom.xml
+ import lzma
+ import tarfile
+ from io import BytesIO
+
+ tar_content = BytesIO()
+ with tarfile.open(fileobj=tar_content, mode="w") as tar:
+ # Add a dummy jar file
+ info = tarfile.TarInfo("test-0.1.0.jar")
+ dummy_jar_content = b"dummy jar content"
+ info.size = len(dummy_jar_content)
+ tar.addfile(info, BytesIO(dummy_jar_content))
+
+ # Add a pom.xml file
+ info = tarfile.TarInfo("pom.xml")
+ dummy_pom_content = b"<project>...</project>"
+ info.size = len(dummy_pom_content)
+ tar.addfile(info, BytesIO(dummy_pom_content))
+
+ tar_content.seek(0)
+ xz_content = BytesIO()
+ with lzma.open(xz_content, "w") as xz:
+ xz.write(tar_content.read())
+
+ xz_content.seek(0)
+ tar_xz_content = xz_content.read()
+
+ librarian = getUtility(ILibraryFileAliasSet)
+ lfa = librarian.create(
+ "archive.tar.xz",
+ len(tar_xz_content),
+ BytesIO(tar_xz_content),
+ "application/x-xz",
+ )
+
+ removeSecurityProxy(self.build).addFile(lfa)
+
+ job = getUtility(ICraftPublishingJobSource).create(self.build)
+ JobRunner([job]).runAll()
+ job = removeSecurityProxy(job)
+
+ self.assertEqual(JobStatus.FAILED, job.job.status)
+ self.assertEqual(
+ "Missing Maven publishing repository configuration",
+ job.error_message,
+ )
diff --git a/lib/lp/services/config/schema-lazr.conf b/lib/lp/services/config/schema-lazr.conf
index 1f0a62f..5dd3279 100644
--- a/lib/lp/services/config/schema-lazr.conf
+++ b/lib/lp/services/config/schema-lazr.conf
@@ -303,13 +303,33 @@ scan_malware: False
[craftbuild.soss]
# value is a JSON Object
+# needed for publishing to artifactory
+# For Maven publishing, the following variables are required:
+# - MAVEN_PUBLISH_URL: URL of the Maven repository to publish to
+# - MAVEN_PUBLISH_AUTH: Authentication credentials for the Maven repository
+#
+# For Cargo publishing, the following variables are required:
+# - CARGO_PUBLISH_URL: URL of the Cargo registry to publish to
+# - CARGO_PUBLISH_AUTH: Authentication token for the Cargo registry
+# - CARGO_REGISTRY_GLOBAL_CREDENTIAL_PROVIDERS: "cargo:token"
+#
+# The following variables can be adjusted to use different Artifactory repositories:
+# - CARGO_ARTIFACTORY1_URL: URL of the Cargo registry to publish to
+# - CARGO_ARTIFACTORY1_READ_AUTH: Authentication credentials for the Cargo registry
+# - MAVEN_ARTIFACTORY1_URL: URL of the Maven repository to publish to
+# - MAVEN_ARTIFACTORY1_READ_AUTH: Authentication credentials for the Maven repository
+#
# example:
# environment_variables: {
# "CARGO_ARTIFACTORY1_URL": "https://canonical.example.com/artifactory/api/cargo/cargo-upstream/index/";,
# "CARGO_ARTIFACTORY1_READ_AUTH": "%(read_auth)s",
+# "CARGO_PUBLISH_URL": "https://canonical.example.com/artifactory/api/cargo/cargo-testing-local/";,
+# "CARGO_PUBLISH_AUTH": "%(write_auth)s",
# "CARGO_REGISTRY_GLOBAL_CREDENTIAL_PROVIDERS": "cargo:token",
# "MAVEN_ARTIFACTORY1_URL": "https://canonical.example.com/artifactory/api/maven/maven-upstream/index";,
# "MAVEN_ARTIFACTORY1_READ_AUTH": "%(read_auth)s",
+# "MAVEN_PUBLISH_URL": "https://canonical.example.com/artifactory/api/maven/maven-testing-local/";,
+# "MAVEN_PUBLISH_AUTH": "%(write_auth)s"
# }
#
environment_variables: none
@@ -2022,6 +2042,11 @@ module: lp.crafts.interfaces.craftrecipejob
dbuser: craft-build-job
crontab_group: MAIN
+[ICraftPublishingJobSource]
+module: lp.crafts.interfaces.craftrecipejob
+dbuser: craft-publish-job
+crontab_group: MAIN
+
[ICIBuildUploadJobSource]
module: lp.soyuz.interfaces.archivejob
dbuser: uploader
diff --git a/lib/lp/services/webhooks/interfaces.py b/lib/lp/services/webhooks/interfaces.py
index f281e3d..166c503 100644
--- a/lib/lp/services/webhooks/interfaces.py
+++ b/lib/lp/services/webhooks/interfaces.py
@@ -58,6 +58,7 @@ WEBHOOK_EVENT_TYPES = {
"merge-proposal:0.1": "Merge proposal",
"oci-recipe:build:0.1": "OCI recipe build",
"snap:build:0.1": "Snap build",
+ "craft-recipe:build:0.1": "Craft recipe build",
}
Follow ups
