launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #32345
[Merge] ~ruinedyourlife/launchpad:native-tools-publishing into launchpad:master
Quentin Debhi has proposed merging ~ruinedyourlife/launchpad:native-tools-publishing into launchpad:master.
Commit message:
Publish maven and cargo artifacts with native tooling for craft builds
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~ruinedyourlife/launchpad/+git/launchpad/+merge/483691
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~ruinedyourlife/launchpad:native-tools-publishing into launchpad:master.
diff --git a/lib/lp/crafts/configure.zcml b/lib/lp/crafts/configure.zcml
index a13bd3f..db94bff 100644
--- a/lib/lp/crafts/configure.zcml
+++ b/lib/lp/crafts/configure.zcml
@@ -63,6 +63,11 @@
interface="lp.crafts.interfaces.craftrecipebuild.ICraftRecipeBuildAdmin" />
</class>
+ <subscriber
+ for="lp.crafts.interfaces.craftrecipebuild.ICraftRecipeBuild
+ lp.crafts.interfaces.craftrecipebuild.ICraftRecipeBuildStatusChangedEvent"
+ handler="lp.crafts.subscribers.craftrecipebuild.craft_build_status_changed" />
+
<!-- CraftRecipeBuildSet -->
<lp:securedutility
class="lp.crafts.model.craftrecipebuild.CraftRecipeBuildSet"
diff --git a/lib/lp/crafts/interfaces/craftrecipejob.py b/lib/lp/crafts/interfaces/craftrecipejob.py
index addc6ee..a5689f2 100644
--- a/lib/lp/crafts/interfaces/craftrecipejob.py
+++ b/lib/lp/crafts/interfaces/craftrecipejob.py
@@ -136,3 +136,45 @@ class ICraftRecipeRequestBuildsJobSource(IJobSource):
recipe and ID, or its `job_type` is not
`CraftRecipeJobType.REQUEST_BUILDS`.
"""
+
+
+class IRustCrateUploadJob(Interface):
+ """A job that uploads a Rust crate to a registry."""
+
+ build_id = Attribute("The ID of the build to upload.")
+ build = Attribute("The build to upload.")
+ error_message = Attribute("The error message if the upload failed.")
+
+ def create(build):
+ """Create a new RustCrateUploadJob."""
+
+
+class IRustCrateUploadJobSource(IJobSource):
+ """A source for creating and finding RustCrateUploadJobs."""
+
+ def create(build):
+ """Upload a Rust crate build to a registry.
+
+ :param build: The build to upload.
+ """
+
+
+class IMavenArtifactUploadJob(Interface):
+ """A job that uploads a Maven artifact to a repository."""
+
+ build_id = Attribute("The ID of the build to upload.")
+ build = Attribute("The build to upload.")
+ error_message = Attribute("The error message if the upload failed.")
+
+ def create(build):
+ """Create a new MavenArtifactUploadJob."""
+
+
+class IMavenArtifactUploadJobSource(IJobSource):
+ """A source for creating and finding MavenArtifactUploadJobs."""
+
+ def create(build):
+ """Upload a Maven artifact build to a repository.
+
+ :param build: The build to upload.
+ """
diff --git a/lib/lp/crafts/model/craftrecipejob.py b/lib/lp/crafts/model/craftrecipejob.py
index d573665..41be4d8 100644
--- a/lib/lp/crafts/model/craftrecipejob.py
+++ b/lib/lp/crafts/model/craftrecipejob.py
@@ -7,8 +7,18 @@ __all__ = [
"CraftRecipeJob",
"CraftRecipeJobType",
"CraftRecipeRequestBuildsJob",
+ "RustCrateUploadJob",
+ "MavenArtifactUploadJob",
]
+import json
+import lzma
+import os
+import subprocess
+import tempfile
+from configparser import NoSectionError
+from tarfile import TarFile
+
import transaction
from lazr.delegates import delegate_to
from lazr.enum import DBEnumeratedType, DBItem
@@ -28,8 +38,15 @@ from lp.crafts.interfaces.craftrecipejob import (
ICraftRecipeJob,
ICraftRecipeRequestBuildsJob,
ICraftRecipeRequestBuildsJobSource,
+ IMavenArtifactUploadJob,
+ IMavenArtifactUploadJobSource,
+ IRustCrateUploadJob,
+ IRustCrateUploadJobSource,
)
from lp.crafts.model.craftrecipebuild import CraftRecipeBuild
+from lp.registry.interfaces.distributionsourcepackage import (
+ IDistributionSourcePackage,
+)
from lp.registry.interfaces.person import IPersonSet
from lp.services.config import config
from lp.services.database.bulk import load_related
@@ -56,6 +73,24 @@ class CraftRecipeJobType(DBEnumeratedType):
""",
)
+ RUST_CRATE_UPLOAD = DBItem(
+ 1,
+ """
+ Rust crate upload
+
+ This job uploads a Rust crate to a registry.
+ """,
+ )
+
+ MAVEN_ARTIFACT_UPLOAD = DBItem(
+ 2,
+ """
+ Maven artifact upload
+
+ This job uploads a Maven artifact to a repository.
+ """,
+ )
+
@implementer(ICraftRecipeJob)
class CraftRecipeJob(StormBase):
@@ -337,3 +372,419 @@ class CraftRecipeRequestBuildsJob(CraftRecipeJobDerived):
# are to this job's metadata and should be preserved.
transaction.commit()
raise
+
+
+@implementer(IRustCrateUploadJob)
+@provider(IRustCrateUploadJobSource)
+class RustCrateUploadJob(CraftRecipeJobDerived):
+ """A Job that uploads a Rust crate to a registry."""
+
+ class_job_type = CraftRecipeJobType.RUST_CRATE_UPLOAD
+
+ user_error_types = ()
+ retry_error_types = ()
+ max_retries = 5
+
+ config = config.IRustCrateUploadJobSource
+
+ @classmethod
+ def create(cls, build):
+ """See `IRustCrateUploadJobSource`."""
+ cls.metadata = {
+ "build_id": build.id,
+ }
+ recipe_job = CraftRecipeJob(
+ build.recipe, cls.class_job_type, cls.metadata
+ )
+ job = cls(recipe_job)
+ job.celeryRunOnCommit()
+ IStore(CraftRecipeJob).flush()
+ return job
+
+ @property
+ def build_id(self):
+ """See `IRustCrateUploadJob`."""
+ return self.metadata["build_id"]
+
+ @cachedproperty
+ def build(self):
+ """See `IRustCrateUploadJob`."""
+ return IStore(CraftRecipeBuild).get(CraftRecipeBuild, self.build_id)
+
+ @property
+ def error_message(self):
+ """See `IRustCrateUploadJob`."""
+ return self.metadata.get("error_message")
+
+ @error_message.setter
+ def error_message(self, message):
+ """See `IRustCrateUploadJob`."""
+ self.metadata["error_message"] = message
+
+ def run(self):
+ """See `IRunnableJob`."""
+ try:
+ # Find the archive file in the build
+ archive_file = None
+ for _, lfa, _ in self.build.getFiles():
+ if lfa.filename.endswith(".tar.xz"):
+ archive_file = lfa
+ break
+
+ if archive_file is None:
+ # Nothing to do
+ self.error_message = "No archive file found in build"
+ return
+
+ # Get the distribution name to access the correct configuration
+ distribution_name = None
+ if (
+ self.build.recipe.git_repository is not None
+ and IDistributionSourcePackage.providedBy(
+ self.build.recipe.git_repository.target
+ )
+ ):
+ distribution_name = (
+ self.build.recipe.git_repository.target.distribution.name
+ )
+
+ if not distribution_name:
+ self.error_message = (
+ "Could not determine distribution for build"
+ )
+ return
+
+ # Get environment variables from configuration
+ try:
+ env_vars_json = config["craftbuild." + distribution_name][
+ "environment_variables"
+ ]
+ if env_vars_json and env_vars_json.lower() != "none":
+ env_vars = json.loads(env_vars_json)
+ # Replace auth placeholders
+ for key, value in env_vars.items():
+ if (
+ isinstance(value, str)
+ and "%(write_auth)s" in value
+ ):
+ env_vars[key] = value.replace(
+ "%(write_auth)s",
+ config.artifactory.write_credentials,
+ )
+ else:
+ env_vars = {}
+ except (NoSectionError, KeyError):
+ self.error_message = (
+ f"No configuration found for {distribution_name}"
+ )
+ return
+
+ # Look for specific Cargo publishing repository configuration
+ cargo_publish_url = env_vars.get("CARGO_PUBLISH_URL")
+ cargo_publish_auth = env_vars.get("CARGO_PUBLISH_AUTH")
+
+ if not cargo_publish_url or not cargo_publish_auth:
+ self.error_message = (
+ "Missing Cargo publishing repository configuration"
+ )
+ return
+
+ # Download and extract the archive to a temporary location
+ with tempfile.TemporaryDirectory() as tmpdir:
+ archive_path = os.path.join(tmpdir, archive_file.filename)
+ with open(archive_path, "wb") as f:
+ with archive_file.open() as lfa_file:
+ f.write(lfa_file.read())
+
+ # Extract the archive
+ extract_dir = os.path.join(tmpdir, "extract")
+ os.makedirs(extract_dir, exist_ok=True)
+ with lzma.open(archive_path) as xz:
+ with TarFile.open(fileobj=xz) as tar:
+ tar.extractall(path=extract_dir)
+
+ # Find the crate file in the extracted archive
+ crate_file = None
+ for root, _, files in os.walk(extract_dir):
+ for filename in files:
+ if filename.endswith(".crate"):
+ crate_file = os.path.join(root, filename)
+ break
+ if crate_file:
+ break
+
+ if crate_file is None:
+ self.error_message = "No .crate file found in archive"
+ return
+
+ # Set up cargo config
+ cargo_dir = os.path.join(tmpdir, ".cargo")
+ os.makedirs(cargo_dir, exist_ok=True)
+
+ # Create config.toml
+ with open(os.path.join(cargo_dir, "config.toml"), "w") as f:
+ f.write(
+ """
+ [registry]
+ global-credential-providers = ["cargo:token"]
+
+ [registries.launchpad]
+ index = "{}"
+ """.format(
+ cargo_publish_url
+ )
+ )
+
+ # Create credentials.toml
+ with open(
+ os.path.join(cargo_dir, "credentials.toml"), "w"
+ ) as f:
+ f.write(
+ """
+ [registries.launchpad]
+ token = "Bearer {}"
+ """.format(
+ cargo_publish_auth
+ )
+ )
+
+ # Run cargo publish
+ result = subprocess.run(
+ [
+ "cargo",
+ "publish",
+ "--no-verify",
+ "--allow-dirty",
+ "--registry",
+ "launchpad",
+ ],
+ cwd=os.path.dirname(crate_file),
+ env={"CARGO_HOME": cargo_dir},
+ capture_output=True,
+ text=True,
+ )
+
+ if result.returncode != 0:
+ self.error_message = (
+ f"Failed to publish crate: {result.stderr}"
+ )
+ raise Exception(self.error_message)
+
+ # Update metadata to indicate successful upload
+ self.error_message = None
+
+ except Exception as e:
+ self.error_message = str(e)
+ # The normal job infrastructure will abort the transaction, but
+ # we want to commit instead: the only database changes we make
+ # are to this job's metadata and should be preserved.
+ transaction.commit()
+ raise
+
+
+@implementer(IMavenArtifactUploadJob)
+@provider(IMavenArtifactUploadJobSource)
+class MavenArtifactUploadJob(CraftRecipeJobDerived):
+ """A Job that uploads a Maven artifact to a repository."""
+
+ class_job_type = CraftRecipeJobType.MAVEN_ARTIFACT_UPLOAD
+
+ user_error_types = ()
+ retry_error_types = ()
+ max_retries = 5
+
+ config = config.IMavenArtifactUploadJobSource
+
+ @classmethod
+ def create(cls, build):
+ """See `IMavenArtifactUploadJobSource`."""
+ cls.metadata = {
+ "build_id": build.id,
+ }
+ recipe_job = CraftRecipeJob(
+ build.recipe, cls.class_job_type, cls.metadata
+ )
+ job = cls(recipe_job)
+ job.celeryRunOnCommit()
+ IStore(CraftRecipeJob).flush()
+ return job
+
+ @property
+ def build_id(self):
+ """See `IMavenArtifactUploadJob`."""
+ return self.metadata["build_id"]
+
+ @cachedproperty
+ def build(self):
+ """See `IMavenArtifactUploadJob`."""
+ return IStore(CraftRecipeBuild).get(CraftRecipeBuild, self.build_id)
+
+ @property
+ def error_message(self):
+ """See `IMavenArtifactUploadJob`."""
+ return self.metadata.get("error_message")
+
+ @error_message.setter
+ def error_message(self, message):
+ """See `IMavenArtifactUploadJob`."""
+ self.metadata["error_message"] = message
+
+ def run(self):
+ """See `IRunnableJob`."""
+ try:
+ # Find the archive file in the build
+ archive_file = None
+ for _, lfa, _ in self.build.getFiles():
+ if lfa.filename.endswith(".tar.xz"):
+ archive_file = lfa
+ break
+
+ if archive_file is None:
+ # Nothing to do
+ self.error_message = "No archive file found in build"
+ return
+
+ # Get the distribution name to access the correct configuration
+ distribution_name = None
+ if (
+ self.build.recipe.git_repository is not None
+ and IDistributionSourcePackage.providedBy(
+ self.build.recipe.git_repository.target
+ )
+ ):
+ distribution_name = (
+ self.build.recipe.git_repository.target.distribution.name
+ )
+
+ if not distribution_name:
+ self.error_message = (
+ "Could not determine distribution for build"
+ )
+ return
+
+ # Get environment variables from configuration
+ try:
+ env_vars_json = config["craftbuild." + distribution_name][
+ "environment_variables"
+ ]
+ if env_vars_json and env_vars_json.lower() != "none":
+ env_vars = json.loads(env_vars_json)
+ # Replace auth placeholders
+ for key, value in env_vars.items():
+ if (
+ isinstance(value, str)
+ and "%(write_auth)s" in value
+ ):
+ env_vars[key] = value.replace(
+ "%(write_auth)s",
+ config.artifactory.write_credentials,
+ )
+ else:
+ env_vars = {}
+ except (NoSectionError, KeyError):
+ self.error_message = (
+ f"No configuration found for {distribution_name}"
+ )
+ return
+
+ # Look for specific Maven publishing repository configuration
+ maven_publish_url = env_vars.get("MAVEN_PUBLISH_URL")
+ maven_publish_auth = env_vars.get("MAVEN_PUBLISH_AUTH")
+
+ if not maven_publish_url or not maven_publish_auth:
+ self.error_message = (
+ "Missing Maven publishing repository configuration"
+ )
+ return
+
+ # Download and extract the archive to a temporary location
+ with tempfile.TemporaryDirectory() as tmpdir:
+ archive_path = os.path.join(tmpdir, archive_file.filename)
+ with open(archive_path, "wb") as f:
+ with archive_file.open() as lfa_file:
+ f.write(lfa_file.read())
+
+ # Extract the archive
+ extract_dir = os.path.join(tmpdir, "extract")
+ os.makedirs(extract_dir, exist_ok=True)
+ with lzma.open(archive_path) as xz:
+ with TarFile.open(fileobj=xz) as tar:
+ tar.extractall(path=extract_dir)
+
+ # Find the JAR file and pom.xml in the extracted archive
+ jar_file = None
+ pom_file = None
+
+ for root, _, files in os.walk(extract_dir):
+ for filename in files:
+ if filename.endswith(".jar"):
+ jar_file = os.path.join(root, filename)
+ elif filename == "pom.xml":
+ pom_file = os.path.join(root, filename)
+ if jar_file is None:
+ self.error_message = "No .jar file found in archive"
+ return
+
+ if pom_file is None:
+ self.error_message = "No pom.xml file found in archive"
+ return
+
+ # Set up Maven settings
+ maven_dir = os.path.join(tmpdir, ".m2")
+ os.makedirs(maven_dir, exist_ok=True)
+
+ # Extract username and password from auth string
+ if ":" in maven_publish_auth:
+ username, password = maven_publish_auth.split(":", 1)
+ else:
+ username = "token"
+ password = maven_publish_auth
+
+ # Create settings.xml with server configuration for the
+ # publishing repository
+ settings_xml = f"""<settings>
+ <servers>
+ <server>
+ <id>launchpad-publish</id>
+ <username>{username}</username>
+ <password>{password}</password>
+ </server>
+ </servers>
+</settings>"""
+
+ with open(os.path.join(maven_dir, "settings.xml"), "w") as f:
+ f.write(settings_xml)
+
+ # Run mvn deploy using the pom file
+ result = subprocess.run(
+ [
+ "mvn",
+ "deploy:deploy-file",
+ f"-DpomFile={pom_file}",
+ f"-Dfile={jar_file}",
+ "-DrepositoryId=launchpad-publish",
+ f"-Durl={maven_publish_url}",
+ "--settings={}".format(
+ os.path.join(maven_dir, "settings.xml")
+ ),
+ ],
+ capture_output=True,
+ text=True,
+ )
+
+ if result.returncode != 0:
+ self.error_message = (
+ f"Failed to publish Maven artifact: {result.stderr}"
+ )
+ raise Exception(self.error_message)
+
+ # Update metadata to indicate successful upload
+ self.error_message = None
+
+ except Exception as e:
+ self.error_message = str(e)
+ # The normal job infrastructure will abort the transaction, but
+ # we want to commit instead: the only database changes we make
+ # are to this job's metadata and should be preserved.
+ transaction.commit()
+ raise
diff --git a/lib/lp/crafts/subscribers/__init__.py b/lib/lp/crafts/subscribers/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/lib/lp/crafts/subscribers/__init__.py
diff --git a/lib/lp/crafts/subscribers/craftrecipebuild.py b/lib/lp/crafts/subscribers/craftrecipebuild.py
new file mode 100644
index 0000000..039d4e4
--- /dev/null
+++ b/lib/lp/crafts/subscribers/craftrecipebuild.py
@@ -0,0 +1,108 @@
+# lib/lp/crafts/subscribers/craftrecipebuild.py
+
+"""Event subscribers for craft recipe builds."""
+
+import lzma
+from configparser import NoSectionError
+from tarfile import TarFile
+
+from zope.component import getUtility
+
+from lp.buildmaster.enums import BuildStatus
+from lp.crafts.interfaces.craftrecipebuild import ICraftRecipeBuild
+from lp.crafts.interfaces.craftrecipejob import (
+ IMavenArtifactUploadJobSource,
+ IRustCrateUploadJobSource,
+)
+from lp.registry.interfaces.distributionsourcepackage import (
+ IDistributionSourcePackage,
+)
+from lp.services.config import config
+from lp.services.scripts import log
+from lp.services.webapp.publisher import canonical_url
+from lp.services.webhooks.interfaces import IWebhookSet
+from lp.services.webhooks.payload import compose_webhook_payload
+
+
+def _trigger_craft_build_webhook(build, action):
+ """Trigger a webhook for a craft recipe build event."""
+ payload = {
+ "craft_build": canonical_url(build, force_local_path=True),
+ "action": action,
+ }
+ payload.update(
+ compose_webhook_payload(
+ ICraftRecipeBuild,
+ build,
+ ["recipe", "build_request", "status"],
+ )
+ )
+ getUtility(IWebhookSet).trigger(
+ build.recipe, "craft-recipe:build:0.1", payload
+ )
+
+
+def craft_build_status_changed(build, event):
+ """Trigger events when craft recipe build statuses change."""
+ _trigger_craft_build_webhook(build, "status-changed")
+
+ if build.status == BuildStatus.FULLYBUILT:
+ # Check if this build is from a configured distribution
+ should_publish = False
+ if (
+ build.recipe.git_repository is not None
+ and IDistributionSourcePackage.providedBy(
+ build.recipe.git_repository.target
+ )
+ ):
+ distribution_name = (
+ build.recipe.git_repository.target.distribution.name
+ )
+ try:
+ # Check if there are any config variables for this distribution
+ config["craftbuild." + distribution_name]
+ should_publish = True
+ except NoSectionError:
+ pass
+
+ # Only schedule uploads for configured distribution builds
+ if should_publish and build.recipe.store_upload:
+ # Get the archive file and check its contents
+ for _, lfa, _ in build.getFiles():
+ if lfa.filename.endswith(".tar.xz"):
+ has_crate, has_jar = check_archive_contents(lfa)
+
+ if has_crate:
+ log.info(
+ "Scheduling upload of Rust crate from %r" % build
+ )
+ getUtility(IRustCrateUploadJobSource).create(build)
+
+ if has_jar:
+ log.info(
+ "Scheduling upload of Maven artifact from %r"
+ % build
+ )
+ getUtility(IMavenArtifactUploadJobSource).create(build)
+
+ break
+
+
+def check_archive_contents(lfa):
+ """Check archive for crates and jars.
+
+ Returns a tuple of (has_crate, has_jar)
+ """
+ has_crate = False
+ has_jar = False
+
+ with lzma.open(lfa.open()) as xz:
+ with TarFile.open(fileobj=xz) as tar:
+ for member in tar.getmembers():
+ if member.name.endswith(".crate"):
+ has_crate = True
+ break
+ elif member.name.endswith(".jar"):
+ has_jar = True
+ break
+ return has_crate, has_jar
diff --git a/lib/lp/services/config/schema-lazr.conf b/lib/lp/services/config/schema-lazr.conf
index 1f0a62f..bfcc5d7 100644
--- a/lib/lp/services/config/schema-lazr.conf
+++ b/lib/lp/services/config/schema-lazr.conf
@@ -303,13 +303,33 @@ scan_malware: False
[craftbuild.soss]
# value is a JSON Object
+# needed for publishing to artifactory
+# For Maven publishing, the following variables are required:
+# - MAVEN_PUBLISH_URL: URL of the Maven repository to publish to
+# - MAVEN_PUBLISH_AUTH: Authentication credentials for the Maven repository
+#
+# For Cargo publishing, the following variables are required:
+# - CARGO_PUBLISH_URL: URL of the Cargo registry to publish to
+# - CARGO_PUBLISH_AUTH: Authentication token for the Cargo registry
+# - CARGO_REGISTRY_GLOBAL_CREDENTIAL_PROVIDERS: "cargo:token"
+#
+# The following variables can be adjusted to use different Artifactory repositories:
+# - CARGO_ARTIFACTORY1_URL: URL of the Cargo registry to publish to
+# - CARGO_ARTIFACTORY1_READ_AUTH: Authentication credentials for the Cargo registry
+# - MAVEN_ARTIFACTORY1_URL: URL of the Maven repository to publish to
+# - MAVEN_ARTIFACTORY1_READ_AUTH: Authentication credentials for the Maven repository
+#
# example:
# environment_variables: {
# "CARGO_ARTIFACTORY1_URL": "https://canonical.example.com/artifactory/api/cargo/cargo-upstream/index/";,
# "CARGO_ARTIFACTORY1_READ_AUTH": "%(read_auth)s",
+# "CARGO_PUBLISH_URL": "https://canonical.example.com/artifactory/api/cargo/lws-cargo-testing-local/";,
+# "CARGO_PUBLISH_AUTH": "%(write_auth)s",
# "CARGO_REGISTRY_GLOBAL_CREDENTIAL_PROVIDERS": "cargo:token",
# "MAVEN_ARTIFACTORY1_URL": "https://canonical.example.com/artifactory/api/maven/maven-upstream/index";,
# "MAVEN_ARTIFACTORY1_READ_AUTH": "%(read_auth)s",
+# "MAVEN_PUBLISH_URL": "https://canonical.example.com/artifactory/api/maven/lws-maven-testing-local/";,
+# "MAVEN_PUBLISH_AUTH": "%(write_auth)s"
# }
#
environment_variables: none
diff --git a/lib/lp/services/webhooks/interfaces.py b/lib/lp/services/webhooks/interfaces.py
index f281e3d..166c503 100644
--- a/lib/lp/services/webhooks/interfaces.py
+++ b/lib/lp/services/webhooks/interfaces.py
@@ -58,6 +58,7 @@ WEBHOOK_EVENT_TYPES = {
"merge-proposal:0.1": "Merge proposal",
"oci-recipe:build:0.1": "OCI recipe build",
"snap:build:0.1": "Snap build",
+ "craft-recipe:build:0.1": "Craft recipe build",
}
Follow ups
