launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #32809
[Merge] ~ruinedyourlife/launchpad-buildd:maven-virtual-remote into launchpad-buildd:master
Quentin Debhi has proposed merging ~ruinedyourlife/launchpad-buildd:maven-virtual-remote into launchpad-buildd:master.
Commit message:
Allow maven to pull plugins from AF and disallow external access
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~ruinedyourlife/launchpad-buildd/+git/launchpad-buildd/+merge/490002
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~ruinedyourlife/launchpad-buildd:maven-virtual-remote into launchpad-buildd:master.
diff --git a/lpbuildd/target/build_craft.py b/lpbuildd/target/build_craft.py
index 69106a8..713795f 100644
--- a/lpbuildd/target/build_craft.py
+++ b/lpbuildd/target/build_craft.py
@@ -160,193 +160,180 @@ class BuildCraft(
def setup_cargo_credentials(self):
"""Set up Cargo credentials through environment variables.
-
+
Transforms input environment variables from:
CARGO_ARTIFACTORY1_URL=https://...
CARGO_ARTIFACTORY1_READ_AUTH=user:token123
-
+
Into Cargo registry environment variables:
CARGO_REGISTRIES_ARTIFACTORY1_INDEX=https://...
CARGO_REGISTRIES_ARTIFACTORY1_TOKEN=Bearer token123
CARGO_REGISTRY_GLOBAL_CREDENTIAL_PROVIDERS=cargo:token
-
- This allows Cargo to authenticate with private registries while building.
+
+ This allows Cargo to authenticate with private registries.
"""
env_vars = dict(
- pair.split("=", maxsplit=1)
+ pair.split("=", maxsplit=1)
for pair in self.args.environment_variables
)
-
+
cargo_vars = {}
-
+
# Process CARGO_* variables into CARGO_REGISTRIES_* format
for key, value in env_vars.items():
if not key.startswith("CARGO_"):
continue
-
+
# Extract name by removing CARGO_ prefix
# and the suffix(_URL or _READ_AUTH)
if key.endswith("_URL"):
# Remove CARGO_ prefix and _URL suffix
- name = key[len("CARGO_"):-len("_URL")]
+ name = key[len("CARGO_") : -len("_URL")]
# Convert URL to registry index
cargo_vars[f"CARGO_REGISTRIES_{name}_INDEX"] = value
elif key.endswith("_READ_AUTH"):
# Remove CARGO_ prefix and _READ_AUTH suffix
- name = key[len("CARGO_"):-len("_READ_AUTH")]
+ name = key[len("CARGO_") : -len("_READ_AUTH")]
# Extract token from user:token format
token = value.split(":", 1)[1]
- cargo_vars[f"CARGO_REGISTRIES_{name}_TOKEN"] = f"Bearer {token}"
+ cargo_vars[f"CARGO_REGISTRIES_{name}_TOKEN"] = (
+ f"Bearer {token}"
+ )
if cargo_vars:
# Tell Cargo to use token-based authentication from environment
- cargo_vars["CARGO_REGISTRY_GLOBAL_CREDENTIAL_PROVIDERS"] = "cargo:token"
-
+ cargo_vars["CARGO_REGISTRY_GLOBAL_CREDENTIAL_PROVIDERS"] = (
+ "cargo:token"
+ )
+
return cargo_vars
def setup_maven_credentials(self):
- """Set up Maven credential files if needed.
-
+ """Set up Maven credential files for Artifactory virtual repository.
+
Creates a Maven settings.xml file with:
- 1. Server configurations for authentication
- - One for release repository
- - One for snapshot repository
- - Uses credentials from MAVEN_*_READ_AUTH variables
-
- 2. Profile configurations for each repository
- - Defines both release and snapshot repositories
- - Configures plugin repositories
- - Controls snapshot behavior
-
- 3. Mirror configurations
- - Redirects all repository requests through Artifactory
- - Ensures dependencies are fetched through our proxy
-
- The resulting settings.xml allows Maven to:
- - Authenticate with private repositories
- - Use the correct URLs for artifacts
- - Handle both release and snapshot dependencies
- - Use plugin repositories with authentication
-
- It is up to the user to activate the profile they want to use
- in the command line (this is done inside sourcecraft.yaml).
+ 1. Server configuration for authentication
+ 2. Profile configuration for the virtual repository
+ 3. Mirror configuration to redirect all requests through Artifactory
+
+ Note: Multi-tenant Maven setups are complex due to how mirrors work.
+ Instead, we use a single Artifactory virtual repository that shadows
+ Maven Central and other remote repositories as needed.
"""
env_vars = dict(
- pair.split("=", maxsplit=1)
+ pair.split("=", maxsplit=1)
for pair in self.args.environment_variables
)
-
+
# Extract Maven-specific variables
- maven_vars = {k: v for k, v in env_vars.items() if k.startswith("MAVEN_")}
+ maven_vars = {
+ k: v for k, v in env_vars.items() if k.startswith("MAVEN_")
+ }
if not maven_vars:
+ logger.info("No Maven environment variables found")
return
# Create .m2 directory for Maven configuration
m2_dir = os.path.join(self.buildd_path, ".m2")
self.backend.run(["mkdir", "-p", m2_dir])
- # Parse repository URLs and credentials from environment variables
- repositories = {}
+ # Parse single repository URL and credentials from env vars
+ repository = {}
for key, value in maven_vars.items():
if key.endswith("_URL"):
- repo_name = key[6:-4].lower() # Remove MAVEN_ and _URL
- repositories.setdefault(repo_name, {})["url"] = value
+ repository["url"] = value
elif key.endswith("_READ_AUTH"):
- repo_name = key[6:-10].lower() # Remove MAVEN_ and _READ_AUTH
user, token = value.split(":")
- repositories.setdefault(repo_name, {})["username"] = user
- repositories.setdefault(repo_name, {})["password"] = token
-
- # Create settings.xml with server configurations
- settings_xml = """<?xml version="1.0" encoding="UTF-8"?>
-<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0";
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd";>
- <servers>
-"""
- # Add authentication for both release and snapshot repositories
- for name, repo in repositories.items():
- if "username" in repo and "password" in repo:
- settings_xml += f""" <server>
- <id>{name}</id>
- <username>{repo['username']}</username>
- <password>{repo['password']}</password>
- </server>
- <server>
- <id>{name}-snapshots</id>
- <username>{repo['username']}</username>
- <password>{repo['password']}</password>
- </server>
-"""
-
- # Add profile configurations for repository definitions
- settings_xml += """ </servers>
- <profiles>
-"""
- for name, repo in repositories.items():
- if "url" in repo:
- # Each profile contains both release and snapshot repositories
- settings_xml += f""" <profile>
- <id>{name}</id>
- <repositories>
- <repository>
- <id>{name}</id>
- <name>{name}</name>
- <url>{repo['url']}</url>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </repository>
- <repository>
- <id>{name}-snapshots</id>
- <name>{name}</name>
- <url>{repo['url']}</url>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </repository>
- </repositories>
- <pluginRepositories>
- <pluginRepository>
- <id>{name}</id>
- <name>{name}</name>
- <url>{repo['url']}</url>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </pluginRepository>
- <pluginRepository>
- <id>{name}-snapshots</id>
- <name>{name}</name>
- <url>{repo['url']}</url>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </pluginRepository>
- </pluginRepositories>
- </profile>
-"""
-
- # Add mirror configurations to ensure all requests go through Artifactory
- settings_xml += """ </profiles>
- <mirrors>
-"""
- for name, repo in repositories.items():
- if "url" in repo:
- settings_xml += f""" <mirror>
- <id>{name}</id>
- <name>Maven Repository Manager running on {name}</name>
- <url>{repo['url']}</url>
- <mirrorOf>*</mirrorOf>
- </mirror>
-"""
-
- settings_xml += """ </mirrors>
-</settings>
-"""
+ repository["username"] = user
+ repository["password"] = token
+
+ # Get repository name from environment variable
+ # (e.g., MAVEN_ARTIFACTORY_URL -> artifactory)
+ repo_name = None
+ for key in maven_vars.keys():
+ if key.endswith("_URL"):
+ repo_name = key[6:-4].lower() # Remove MAVEN_ and _URL
+ break
+
+ if not repo_name or not repository.get("url"):
+ return
+
+ # Generate Maven settings.xml
+ settings_xml = self._generate_maven_settings_xml(repo_name, repository)
+
with self.backend.open(os.path.join(m2_dir, "settings.xml"), "w") as f:
f.write(settings_xml)
+ def _generate_maven_settings_xml(self, repo_name, repository):
+ """Generate Maven settings.xml for single virtual repository.
+
+ :param repo_name: Name/ID of the repository
+ :param repository: Dict with repository configuration
+ :return: XML content as string
+ """
+ # XML header
+ header = (
+ '<?xml version="1.0" encoding="UTF-8"?>\n'
+ '<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"\n'
+ ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"\n'
+ )
+
+ schema = (
+ ' xsi:schemaLocation="'
+ "http://maven.apache.org/SETTINGS/1.0.0 "
+ 'http://maven.apache.org/xsd/settings-1.0.0.xsd";>\n'
+ )
+
+ # Servers section for authentication
+ servers = " <servers>\n"
+ if "username" in repository and "password" in repository:
+ servers += (
+ " <server>\n"
+ f" <id>{repo_name}</id>\n"
+ f" <username>{repository['username']}</username>\n"
+ f" <password>{repository['password']}</password>\n"
+ " </server>\n"
+ )
+ servers += " </servers>\n"
+
+ # Profiles section with virtual repository
+ profiles = " <profiles>\n"
+ profiles += (
+ " <profile>\n"
+ f" <id>{repo_name}</id>\n"
+ " <repositories>\n"
+ " <repository>\n"
+ f" <id>{repo_name}</id>\n"
+ f" <name>Artifactory Virtual Repo</name>\n"
+ f" <url>{repository['url']}</url>\n"
+ " </repository>\n"
+ " </repositories>\n"
+ " <pluginRepositories>\n"
+ " <pluginRepository>\n"
+ f" <id>{repo_name}</id>\n"
+ f" <name>Artifactory Virtual Repo</name>\n"
+ f" <url>{repository['url']}</url>\n"
+ " </pluginRepository>\n"
+ " </pluginRepositories>\n"
+ " </profile>\n"
+ )
+ profiles += " </profiles>\n"
+
+ # Mirrors section - redirect all requests through Artifactory
+ mirrors = (
+ " <mirrors>\n"
+ " <mirror>\n"
+ f" <id>{repo_name}</id>\n"
+ f" <name>Artifactory Virtual Repo</name>\n"
+ f" <url>{repository['url']}</url>\n"
+ " <mirrorOf>*</mirrorOf>\n"
+ " </mirror>\n"
+ " </mirrors>\n"
+ )
+
+ # Combine all parts
+ return header + schema + servers + profiles + mirrors + "</settings>"
+
def build(self):
"""Running build phase..."""
# Set up credential files before building
diff --git a/lpbuildd/target/tests/test_build_craft.py b/lpbuildd/target/tests/test_build_craft.py
index fe342c4..0e83514 100644
--- a/lpbuildd/target/tests/test_build_craft.py
+++ b/lpbuildd/target/tests/test_build_craft.py
@@ -1296,7 +1296,7 @@ class TestBuildCraft(TestCase):
"--branch",
"lp:foo",
"--environment-variable",
- "MAVEN_ARTIFACTORY_URL=https://canonical.example.com/artifactory/api/maven/maven-upstream/";,
+ "MAVEN_ARTIFACTORY_URL=https://canonical.example.com/artifactory/virtual-maven/";,
"--environment-variable",
"MAVEN_ARTIFACTORY_READ_AUTH=user:token123",
"test-image",
@@ -1309,33 +1309,50 @@ class TestBuildCraft(TestCase):
self.assertTrue(build_craft.backend.path_exists(maven_settings_path))
with build_craft.backend.open(maven_settings_path) as f:
settings_content = f.read()
-
+
# Check server configurations
- self.assertIn(' <server>', settings_content)
- self.assertIn(' <id>artifactory</id>', settings_content)
- self.assertIn(' <username>user</username>', settings_content)
- self.assertIn(' <password>token123</password>', settings_content)
- self.assertIn(' <id>artifactory-snapshots</id>', settings_content)
-
+ self.assertIn(" <server>", settings_content)
+ self.assertIn(" <id>artifactory</id>", settings_content)
+ self.assertIn(
+ " <username>user</username>", settings_content
+ )
+ self.assertIn(
+ " <password>token123</password>", settings_content
+ )
+
# Check profile configuration
- self.assertIn(' <profile>', settings_content)
- self.assertIn(' <id>artifactory</id>', settings_content)
-
- # Check repositories
- self.assertIn(' <repository>', settings_content)
- self.assertIn(' <url>https://canonical.example.com/artifactory/api/maven/maven-upstream/</url>', settings_content)
- self.assertIn(' <enabled>false</enabled>', settings_content)
- self.assertIn(' <enabled>true</enabled>', settings_content)
-
- # Check plugin repositories
- self.assertIn(' <pluginRepository>', settings_content)
-
- # Check mirrors
- self.assertIn(' <mirror>', settings_content)
- self.assertIn(' <mirrorOf>*</mirrorOf>', settings_content)
-
- # Verify activeProfiles section is not present
- self.assertNotIn('<activeProfiles>', settings_content)
+ self.assertIn(" <profile>", settings_content)
+ self.assertIn(" <id>artifactory</id>", settings_content)
+
+ # Check repository configuration
+ self.assertIn(" <repository>", settings_content)
+ self.assertIn(
+ " <name>Artifactory Virtual Repo</name>",
+ settings_content,
+ )
+ self.assertIn(
+ " <url>https://canonical.example.com/artifactory/virtual-maven/</url>",
+ settings_content,
+ )
+
+ # Check plugin repository configuration
+ self.assertIn(
+ " <pluginRepository>", settings_content
+ )
+ self.assertIn(
+ " <name>Artifactory Virtual Repo</name>",
+ settings_content,
+ )
+
+ # Check mirror configuration
+ self.assertIn(" <mirror>", settings_content)
+ self.assertIn(
+ " <name>Artifactory Virtual Repo</name>",
+ settings_content,
+ )
+ self.assertIn(
+ " <mirrorOf>*</mirrorOf>", settings_content
+ )
# Verify the build command was run
self.assertThat(
@@ -1351,7 +1368,8 @@ class TestBuildCraft(TestCase):
),
)
- def test_build_with_multiple_maven_credentials(self):
+ def test_build_without_maven_credentials(self):
+ """Test that no Maven settings.xml is created with no Maven vars ."""
args = [
"build-craft",
"--backend=fake",
@@ -1360,56 +1378,20 @@ class TestBuildCraft(TestCase):
"1",
"--branch",
"lp:foo",
- "--environment-variable",
- "MAVEN_ARTIFACTORY1_URL=https://canonical.example.com/artifactory/api/maven/maven-upstream1/";,
- "--environment-variable",
- "MAVEN_ARTIFACTORY1_READ_AUTH=user1:token1",
- "--environment-variable",
- "MAVEN_ARTIFACTORY2_URL=https://canonical.example.com/artifactory/api/maven/maven-upstream2/";,
- "--environment-variable",
- "MAVEN_ARTIFACTORY2_READ_AUTH=user2:token2",
"test-image",
]
build_craft = parse_args(args=args).operation
build_craft.build()
- # Check that .m2/settings.xml was created correctly
+ # Check that .m2/settings.xml was NOT created
maven_settings_path = "/home/buildd/test-image/.m2/settings.xml"
- self.assertTrue(build_craft.backend.path_exists(maven_settings_path))
- with build_craft.backend.open(maven_settings_path) as f:
- settings_content = f.read()
-
- # Check server configurations for both repositories
- for i, creds in enumerate([("user1", "token1"), ("user2", "token2")], 1):
- user, token = creds
- repo_name = f"artifactory{i}"
-
- # Check servers with exact indentation
- self.assertIn(f' <id>{repo_name}</id>', settings_content)
- self.assertIn(f' <username>{user}</username>', settings_content)
- self.assertIn(f' <password>{token}</password>', settings_content)
- self.assertIn(f' <id>{repo_name}-snapshots</id>', settings_content)
-
- # Check profiles with exact indentation
- self.assertIn(f' <id>{repo_name}</id>', settings_content)
-
- # Check repositories with exact indentation
- url = f'https://canonical.example.com/artifactory/api/maven/maven-upstream{i}/'
- self.assertIn(f' <url>{url}</url>', settings_content)
-
- # Check mirrors with exact indentation
- self.assertIn(f' <id>{repo_name}</id>', settings_content)
- self.assertIn(' <mirrorOf>*</mirrorOf>', settings_content)
-
- # Verify activeProfiles section is not present
- self.assertNotIn('<activeProfiles>', settings_content)
+ self.assertFalse(build_craft.backend.path_exists(maven_settings_path))
- # Verify the build command was run
+ # Verify only the build command was run (no mkdir for .m2)
self.assertThat(
build_craft.backend.run.calls,
MatchesListwise(
[
- RanCommand(["mkdir", "-p", "/home/buildd/test-image/.m2"]),
RanBuildCommand(
["sourcecraft", "pack", "-v", "--destructive-mode"],
cwd="/home/buildd/test-image/.",
Follow ups
