launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #32175
[Merge] ~tushar5526/launchpad:add-support-for-parallel-publishing into launchpad:master
Tushar Gupta has proposed merging ~tushar5526/launchpad:add-support-for-parallel-publishing into launchpad:master.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~tushar5526/launchpad/+git/launchpad/+merge/480435
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~tushar5526/launchpad:add-support-for-parallel-publishing into launchpad:master.
diff --git a/lib/lp/archivepublisher/scripts/base.py b/lib/lp/archivepublisher/scripts/base.py
index f590791..757c909 100644
--- a/lib/lp/archivepublisher/scripts/base.py
+++ b/lib/lp/archivepublisher/scripts/base.py
@@ -14,6 +14,7 @@ from zope.component import getUtility
from lp.registry.interfaces.distribution import IDistributionSet
from lp.services.scripts.base import LaunchpadCronScript
+from lp.soyuz.interfaces.archive import IArchiveSet
class PublisherScript(LaunchpadCronScript):
@@ -36,6 +37,34 @@ class PublisherScript(LaunchpadCronScript):
help="Publish all Ubuntu-derived distributions.",
)
+ def addParallelPublisherOptions(self):
+ self.parser.add_option(
+ "--archive",
+ action="append",
+ dest="archives",
+ metavar="REFERENCE",
+ default=[],
+ help="Only run over the archives identified by this reference. "
+ "You can specify multiple archives by repeating the option",
+ )
+
+ self.parser.add_option(
+ "--exclude",
+ action="append",
+ dest="excluded_archives",
+ metavar="REFERENCE",
+ default=[],
+ help="Skip the archives identified by this reference in the "
+ "publisher run. You can specify multiple archives by repeating "
+ "the option",
+ )
+
+ self.parser.add_option(
+ "--lockfilename",
+ dest="lockfilename",
+ help="lockfilename to be used by the publisher run",
+ )
+
def findSelectedDistro(self):
"""Find the `Distribution` named by the --distribution option.
@@ -62,3 +91,35 @@ class PublisherScript(LaunchpadCronScript):
return self.findDerivedDistros()
else:
return [self.findSelectedDistro()]
+
+ def findArchives(self, archive_references, distribution=None):
+ """
+ Retrieve a list of archives based on the provided references and
+ optional distribution.
+
+ Args:
+ archive_references (list): A list of archive references to
+ retrieve.
+ distribution (IDistributionSet, optional): The distribution
+ to filter archives by. Defaults to None.
+
+ Returns:
+ list: A list of archives that match the provided references and
+ distribution.
+ """
+ if not archive_references:
+ return []
+
+ archives = []
+ for reference in archive_references:
+ archive = getUtility(IArchiveSet).getByReference(reference)
+ if not archive:
+ self.logger.warning(
+ "Cannot find the excluded archive with reference: '%s', "
+ % reference
+ )
+ continue
+ if distribution and archive.distribution != distribution:
+ continue
+ archives.append(archive)
+ return archives
diff --git a/lib/lp/archivepublisher/scripts/processaccepted.py b/lib/lp/archivepublisher/scripts/processaccepted.py
index da27fb9..dc46718 100644
--- a/lib/lp/archivepublisher/scripts/processaccepted.py
+++ b/lib/lp/archivepublisher/scripts/processaccepted.py
@@ -40,11 +40,12 @@ class ProcessAccepted(PublisherScript):
@property
def lockfilename(self):
"""See `LaunchpadScript`."""
- return GLOBAL_PUBLISHER_LOCK
+ return self.options.lockfilename or GLOBAL_PUBLISHER_LOCK
def add_my_options(self):
"""Command line options for this script."""
self.addDistroOptions()
+ self.addParallelPublisherOptions()
self.parser.add_option(
"--ppa",
@@ -62,6 +63,18 @@ class ProcessAccepted(PublisherScript):
help="Run only over COPY archives.",
)
+ def countExclusiveOptions(self):
+ """Return the number of exclusive "mode" options that were set.
+
+ In valid use, at most one of them should be set.
+ """
+ exclusive_options = [
+ self.options.ppa,
+ self.options.copy_archives,
+ self.options.archives,
+ ]
+ return len(list(filter(None, exclusive_options)))
+
def validateArguments(self):
"""Validate command-line arguments."""
if self.options.ppa and self.options.copy_archives:
@@ -72,11 +85,25 @@ class ProcessAccepted(PublisherScript):
raise OptionValueError(
"Can't combine --derived with a distribution name."
)
+ if self.countExclusiveOptions() > 1:
+ raise OptionValueError(
+ "Can only specify one of ppa, copy-archive, archive"
+ )
def getTargetArchives(self, distribution):
"""Find archives to target based on given options."""
+ excluded_archives = self.findArchives(
+ self.options.excluded_archives, distribution
+ )
+
+ if self.options.archives:
+ return self.findArchives(self.options.archives, distribution)
if self.options.ppa:
- return distribution.getPendingAcceptancePPAs()
+ return [
+ archive
+ for archive in distribution.getPendingAcceptancePPAs()
+ if archive not in excluded_archives
+ ]
elif self.options.copy_archives:
return getUtility(IArchiveSet).getArchivesForDistribution(
distribution, purposes=[ArchivePurpose.COPY]
diff --git a/lib/lp/archivepublisher/scripts/publishdistro.py b/lib/lp/archivepublisher/scripts/publishdistro.py
index 38ee406..22cc478 100644
--- a/lib/lp/archivepublisher/scripts/publishdistro.py
+++ b/lib/lp/archivepublisher/scripts/publishdistro.py
@@ -71,10 +71,13 @@ def has_oval_data_changed(incoming_dir, published_dir):
class PublishDistro(PublisherScript):
"""Distro publisher."""
- lockfilename = GLOBAL_PUBLISHER_LOCK
+ @property
+ def lockfilename(self):
+ return self.options.lockfilename or GLOBAL_PUBLISHER_LOCK
def add_my_options(self):
self.addDistroOptions()
+ self.addParallelPublisherOptions()
self.parser.add_option(
"-C",
@@ -227,13 +230,6 @@ class PublishDistro(PublisherScript):
help="Only run over the copy archives.",
)
- self.parser.add_option(
- "--archive",
- dest="archive",
- metavar="REFERENCE",
- help="Only run over the archive identified by this reference.",
- )
-
def isCareful(self, option):
"""Is the given "carefulness" option enabled?
@@ -275,7 +271,7 @@ class PublishDistro(PublisherScript):
self.options.ppa,
self.options.private_ppa,
self.options.copy_archive,
- self.options.archive,
+ self.options.archives,
]
return len(list(filter(None, exclusive_options)))
@@ -375,20 +371,32 @@ class PublishDistro(PublisherScript):
def getTargetArchives(self, distribution):
"""Find the archive(s) selected by the script's options."""
- if self.options.archive:
- archive = getUtility(IArchiveSet).getByReference(
- self.options.archive
- )
- if archive.distribution == distribution:
- return [archive]
- else:
- return []
+ if self.options.archives:
+ return self.findArchives(self.options.archives, distribution)
elif self.options.partner:
return [distribution.getArchiveByComponent("partner")]
elif self.options.ppa:
- return filter(is_ppa_public, self.getPPAs(distribution))
+ return [
+ archive
+ for archive in filter(
+ is_ppa_public, self.getPPAs(distribution)
+ )
+ if archive
+ not in self.findArchives(
+ self.options.excluded_archives, distribution
+ )
+ ]
elif self.options.private_ppa:
- return filter(is_ppa_private, self.getPPAs(distribution))
+ return [
+ archive
+ for archive in filter(
+ is_ppa_private, self.getPPAs(distribution)
+ )
+ if archive
+ not in self.findArchives(
+ self.options.excluded_archives, distribution
+ )
+ ]
elif self.options.copy_archive:
return self.getCopyArchives(distribution)
else:
@@ -597,12 +605,14 @@ class PublishDistro(PublisherScript):
# store and cause performance problems.
Store.of(archive).reset()
- def rsyncOVALData(self):
+ def _buildRsyncCommand(self, src, dest, extra_options=None):
+ if extra_options is None:
+ extra_options = []
+
# Ensure that the rsync paths have a trailing slash.
- rsync_src = os.path.join(
- config.archivepublisher.oval_data_rsync_endpoint, ""
- )
- rsync_dest = os.path.join(config.archivepublisher.oval_data_root, "")
+ rsync_src = os.path.join(src, "")
+ rsync_dest = os.path.join(dest, "")
+
rsync_command = [
"/usr/bin/rsync",
"-a",
@@ -612,28 +622,69 @@ class PublishDistro(PublisherScript):
),
"--delete",
"--delete-after",
- rsync_src,
- rsync_dest,
]
- try:
- self.logger.info(
- "Attempting to rsync the OVAL data from '%s' to '%s'",
- rsync_src,
- rsync_dest,
- )
- check_call(rsync_command)
- except CalledProcessError:
- self.logger.exception(
- "Failed to rsync OVAL data from '%s' to '%s'",
- rsync_src,
- rsync_dest,
+ rsync_command.extend(extra_options)
+ rsync_command.extend([rsync_src, rsync_dest])
+ return rsync_command
+
+ def _generateOVALDataRsyncCommands(self):
+ if self.options.archives:
+ return [
+ self._buildRsyncCommand(
+ # -R: copies the OVALData and preserves the src path in
+ # dest directory
+ # --ignore-missing-args: If the source directory is not
+ # present, don't throw an error
+ # man rsync can provide detailed explanation of these
+ # options
+ extra_options=["-R", "--ignore-missing-args"],
+ src=os.path.join(
+ config.archivepublisher.oval_data_rsync_endpoint,
+ reference,
+ ),
+ dest=os.path.join(config.archivepublisher.oval_data_root),
+ )
+ for reference in self.options.archives
+ ]
+
+ exclude_options = []
+ # If there are any archives specified to be excluded, exclude rsync
+ # for them in the rsync command
+ for excluded_archive in self.findArchives(
+ self.options.excluded_archives
+ ):
+ exclude_options.append("--exclude")
+ exclude_options.append(excluded_archive.reference)
+ return [
+ self._buildRsyncCommand(
+ extra_options=exclude_options,
+ src=config.archivepublisher.oval_data_rsync_endpoint,
+ dest=config.archivepublisher.oval_data_root,
)
- raise
+ ]
+
+ def rsyncOVALData(self):
+ for rsync_command in self._generateOVALDataRsyncCommands():
+ try:
+ self.logger.info(
+ "Attempting to rsync the OVAL data: %s",
+ rsync_command,
+ )
+ check_call(rsync_command)
+ except CalledProcessError:
+ self.logger.exception(
+ "Failed to rsync OVAL data: %s",
+ rsync_command,
+ )
+ raise
def checkForUpdatedOVALData(self, distribution):
"""Compare the published OVAL files with the incoming one."""
start_dir = Path(config.archivepublisher.oval_data_root)
archive_set = getUtility(IArchiveSet)
+ excluded_archives = self.findArchives(
+ self.options.excluded_archives, distribution
+ )
for owner_path in start_dir.iterdir():
if not owner_path.name.startswith("~"):
continue
@@ -644,6 +695,11 @@ class PublishDistro(PublisherScript):
archive = archive_set.getPPAByDistributionAndOwnerName(
distribution, owner_path.name[1:], archive_path.name
)
+ if (
+ self.options.archives
+ and archive.reference not in self.options.archives
+ ):
+ continue
if archive is None:
self.logger.info(
"Skipping OVAL data for '~%s/%s/%s' "
@@ -653,6 +709,15 @@ class PublishDistro(PublisherScript):
archive_path.name,
)
continue
+ if archive in excluded_archives:
+ self.logger.info(
+ "Skipping OVAL data for '~%s/%s/%s' "
+ "(archive excluded from the publisher run).",
+ owner_path.name[1:],
+ distribution.name,
+ archive_path.name,
+ )
+ continue
for suite_path in archive_path.iterdir():
try:
series, pocket = distribution.getDistroSeriesAndPocket(
diff --git a/lib/lp/archivepublisher/tests/test_processaccepted.py b/lib/lp/archivepublisher/tests/test_processaccepted.py
index 34c459e..8c95f7a 100644
--- a/lib/lp/archivepublisher/tests/test_processaccepted.py
+++ b/lib/lp/archivepublisher/tests/test_processaccepted.py
@@ -8,6 +8,7 @@ from optparse import OptionValueError
import transaction
from testtools.matchers import EndsWith, LessThan, MatchesListwise
+from lp.archivepublisher.publishing import GLOBAL_PUBLISHER_LOCK
from lp.archivepublisher.scripts.processaccepted import ProcessAccepted
from lp.registry.interfaces.series import SeriesStatus
from lp.services.config import config
@@ -269,3 +270,104 @@ class TestProcessAccepted(TestCaseWithFactory):
]
),
)
+
+ def test_countExclusiveOptions_is_zero_if_none_set(self):
+ # If none of the exclusive options is set, countExclusiveOptions
+ # counts zero.
+ self.assertEqual(0, self.getScript().countExclusiveOptions())
+
+ def test_countExclusiveOptions_counts_ppa(self):
+ # countExclusiveOptions includes the "ppa" option.
+ self.assertEqual(
+ 1, self.getScript(test_args=["--ppa"]).countExclusiveOptions()
+ )
+
+ def test_countExclusiveOptions_counts_copy_archives(self):
+ # countExclusiveOptions includes the "copy-archive" option.
+ self.assertEqual(
+ 1,
+ self.getScript(
+ test_args=["--copy-archives"]
+ ).countExclusiveOptions(),
+ )
+
+ def test_countExclusiveOptions_counts_archive(self):
+ # countExclusiveOptions includes the "copy-archive" option.
+ self.assertEqual(
+ 1, self.getScript(test_args=["--archive"]).countExclusiveOptions()
+ )
+
+ def test_lockfilename_option_overrides_default_lock(self):
+ script = self.getScript(test_args=["--lockfilename", "foo.lock"])
+ assert script.lockfilepath == "/var/lock/foo.lock"
+
+ def test_default_lock(self):
+ script = self.getScript()
+ assert script.lockfilepath == "/var/lock/%s" % GLOBAL_PUBLISHER_LOCK
+
+ def test_getTargetArchives_ignores_excluded_archives_for_ppa(self):
+ # If the selected exclusive option is "ppa," getTargetArchives
+ # leaves out excluded PPAs.
+ distroseries = self.factory.makeDistroSeries(distribution=self.distro)
+
+ ppa = self.factory.makeArchive(self.distro, purpose=ArchivePurpose.PPA)
+ excluded_ppa_1 = self.factory.makeArchive(
+ self.distro, purpose=ArchivePurpose.PPA
+ )
+ excluded_ppa_2 = self.factory.makeArchive(
+ self.distro, purpose=ArchivePurpose.PPA, private=True
+ )
+
+ for archive in [ppa, excluded_ppa_1, excluded_ppa_2]:
+ self.createWaitingAcceptancePackage(
+ archive=archive, distroseries=distroseries
+ )
+ script = self.getScript(
+ test_args=[
+ "--ppa",
+ "--exclude",
+ excluded_ppa_1.reference,
+ "--exclude",
+ excluded_ppa_2.reference,
+ ],
+ )
+ self.assertContentEqual([ppa], script.getTargetArchives(self.distro))
+
+ def test_getTargetArchives_gets_specific_archives(self):
+ # If the selected exclusive option is "archive,"
+ # getTargetArchives looks for the specified archives.
+
+ distroseries = self.factory.makeDistroSeries(distribution=self.distro)
+
+ ppa1 = self.factory.makeArchive(
+ self.distro, purpose=ArchivePurpose.PPA
+ )
+ ppa2 = self.factory.makeArchive(
+ self.distro, purpose=ArchivePurpose.PPA, private=True
+ )
+ ppa3 = self.factory.makeArchive(
+ self.distro, purpose=ArchivePurpose.PPA
+ )
+ ppa4 = self.factory.makeArchive(
+ self.distro, purpose=ArchivePurpose.PPA, private=True
+ )
+
+ # create another random archive in the same distro
+ self.factory.makeArchive(self.distro, purpose=ArchivePurpose.PPA)
+
+ for archive in [ppa1, ppa2, ppa3, ppa4]:
+ self.createWaitingAcceptancePackage(
+ archive=archive, distroseries=distroseries
+ )
+
+ script = self.getScript(
+ test_args=[
+ "--archive",
+ ppa1.reference,
+ "--archive",
+ ppa2.reference,
+ ],
+ )
+ self.assertContentEqual(
+ [ppa1, ppa2], script.getTargetArchives(self.distro)
+ )
diff --git a/lib/lp/archivepublisher/tests/test_publishdistro.py b/lib/lp/archivepublisher/tests/test_publishdistro.py
index ad591fa..a7fa759 100644
--- a/lib/lp/archivepublisher/tests/test_publishdistro.py
+++ b/lib/lp/archivepublisher/tests/test_publishdistro.py
@@ -24,7 +24,7 @@ from lp.archivepublisher.interfaces.archivegpgsigningkey import (
IArchiveGPGSigningKey,
)
from lp.archivepublisher.interfaces.publisherconfig import IPublisherConfigSet
-from lp.archivepublisher.publishing import Publisher
+from lp.archivepublisher.publishing import GLOBAL_PUBLISHER_LOCK, Publisher
from lp.archivepublisher.scripts.publishdistro import PublishDistro
from lp.archivepublisher.tests.artifactory_fixture import (
FakeArtifactoryFixture,
@@ -34,7 +34,7 @@ from lp.registry.interfaces.person import IPersonSet
from lp.registry.interfaces.pocket import PackagePublishingPocket
from lp.services.config import config
from lp.services.database.interfaces import IStore
-from lp.services.log.logger import BufferLogger, DevNullLogger
+from lp.services.log.logger import BufferLogger
from lp.services.osutils import write_file
from lp.services.scripts.base import LaunchpadScriptFailure
from lp.soyuz.enums import (
@@ -330,11 +330,94 @@ class TestPublishDistro(TestNativePublishingBase):
]
mock_subprocess_check_call.assert_called_once_with(call_args)
expected_log_line = (
- "ERROR Failed to rsync OVAL data from "
- "'oval.internal::oval/' to '%s/'" % self.oval_data_root
+ "ERROR Failed to rsync OVAL data: "
+ "['/usr/bin/rsync', '-a', '-q', '--timeout=90', '--delete', "
+ "'--delete-after', 'oval.internal::oval/', '%s/']"
+ % self.oval_data_root
)
self.assertTrue(expected_log_line in self.logger.getLogBuffer())
+ def testPublishDistroOVALDataRsyncForExcludedArchives(self):
+ """
+ Test publisher skips excluded archives specified via --exclude
+ during OVALData rsync.
+ """
+ self.setUpOVALDataRsync()
+ ppa1 = self.factory.makeArchive(private=True)
+ ppa2 = self.factory.makeArchive()
+ self.factory.makeArchive()
+
+ mock_subprocess_check_call = self.useFixture(
+ MockPatch("lp.archivepublisher.scripts.publishdistro.check_call")
+ ).mock
+
+ call_args = [
+ "/usr/bin/rsync",
+ "-a",
+ "-q",
+ "--timeout=90",
+ "--delete",
+ "--delete-after",
+ "--exclude",
+ ppa1.reference,
+ "--exclude",
+ ppa2.reference,
+ "oval.internal::oval/",
+ self.oval_data_root + "/",
+ ]
+ self.runPublishDistro(
+ extra_args=[
+ "--exclude",
+ ppa1.reference,
+ "--exclude",
+ ppa2.reference,
+ ]
+ )
+ mock_subprocess_check_call.assert_called_once_with(call_args)
+
+ def testPublishDistroOVALDataRsyncForSpecificArchives(self):
+ """
+ Test publisher only runs for archives specified via --archive
+ during OVALData rsync.
+ """
+ self.setUpOVALDataRsync()
+ ppa1 = self.factory.makeArchive(private=True)
+ ppa2 = self.factory.makeArchive()
+ self.factory.makeArchive()
+
+ mock_subprocess_check_call = self.useFixture(
+ MockPatch("lp.archivepublisher.scripts.publishdistro.check_call")
+ ).mock
+
+ call_args = [
+ call(
+ [
+ "/usr/bin/rsync",
+ "-a",
+ "-q",
+ "--timeout=90",
+ "--delete",
+ "--delete-after",
+ "-R",
+ "--ignore-missing-args",
+ os.path.join("oval.internal::oval/", ppa.reference, ""),
+ self.oval_data_root + "/",
+ ]
+ )
+ for ppa in [ppa1, ppa2]
+ ]
+
+ self.runPublishDistro(
+ extra_args=[
+ "--archive",
+ ppa1.reference,
+ "--archive",
+ ppa2.reference,
+ ]
+ )
+
+ assert mock_subprocess_check_call.call_args_list == call_args
+
def test_checkForUpdatedOVALData_new(self):
self.setUpOVALDataRsync()
self.useFixture(
@@ -487,6 +570,105 @@ class TestPublishDistro(TestNativePublishingBase):
)
self.assertIsNone(archive.dirty_suites)
+ def test_checkForUpdatedOVALData_skips_excluded_ppas(self):
+ """
+ Skip excluded PPAs in checkForUpdatedOVALData
+ """
+ self.setUpOVALDataRsync()
+ self.useFixture(
+ MockPatch("lp.archivepublisher.scripts.publishdistro.check_call")
+ )
+ ppa1 = self.factory.makeArchive(purpose=ArchivePurpose.PPA)
+ ppa2 = self.factory.makeArchive(purpose=ArchivePurpose.PPA)
+ ppa3 = self.factory.makeArchive(purpose=ArchivePurpose.PPA)
+ # Disable normal publication so that dirty_suites isn't cleared.
+ ppa1.publish = False
+ ppa2.publish = False
+ ppa3.publish = False
+
+ for archive in [ppa1, ppa2, ppa3]:
+ incoming_dir = (
+ Path(self.oval_data_root)
+ / archive.reference
+ / "breezy-autotest"
+ / "main"
+ )
+ write_file(str(incoming_dir), b"test")
+
+ self.runPublishDistro(
+ extra_args=[
+ "--ppa",
+ "--exclude",
+ ppa2.reference,
+ "--exclude",
+ ppa3.reference,
+ ],
+ distribution="ubuntu",
+ )
+
+ self.assertEqual(["breezy-autotest"], ppa1.dirty_suites)
+ self.assertIsNone(ppa2.dirty_suites)
+ self.assertIsNone(ppa3.dirty_suites)
+ self.assertIn(
+ "INFO Skipping OVAL data for '%s' "
+ "(archive excluded from the publisher run)." % (ppa2.reference),
+ self.logger.getLogBuffer(),
+ )
+ self.assertIn(
+ "INFO Skipping OVAL data for '%s' "
+ "(archive excluded from the publisher run)." % (ppa3.reference),
+ self.logger.getLogBuffer(),
+ )
+
+ def test_checkForUpdatedOVALData_runs_for_specific_archive(self):
+ """
+ checkForUpdatedOVALData should only run for specific archives
+ if "archive" option is specified.
+ """
+
+ self.setUpOVALDataRsync()
+ self.useFixture(
+ MockPatch("lp.archivepublisher.scripts.publishdistro.check_call")
+ )
+
+ ppa1 = self.factory.makeArchive(purpose=ArchivePurpose.PPA)
+ ppa2 = self.factory.makeArchive(purpose=ArchivePurpose.PPA)
+ ppa3 = self.factory.makeArchive(purpose=ArchivePurpose.PPA)
+ # Disable normal publication so that dirty_suites isn't cleared.
+ ppa1.publish = False
+ ppa2.publish = False
+ ppa3.publish = False
+
+ for archive in [ppa1, ppa2, ppa3]:
+ incoming_dir = (
+ Path(self.oval_data_root)
+ / archive.reference
+ / "breezy-autotest"
+ / "main"
+ )
+ write_file(str(incoming_dir), b"test")
+
+ self.runPublishDistro(
+ extra_args=[
+ "--archive",
+ ppa1.reference,
+ "--archive",
+ ppa2.reference,
+ ],
+ distribution="ubuntu",
+ )
+
+ self.assertEqual(["breezy-autotest"], ppa1.dirty_suites)
+ self.assertEqual(["breezy-autotest"], ppa2.dirty_suites)
+ self.assertIsNone(ppa3.dirty_suites)
+
+ # Further logs should not have any reference to other PPAs
+ # as we skip them when --archive option is set.
+ self.assertNotIn(
+ ppa3.reference,
+ self.logger.getLogBuffer(),
+ )
+
@defer.inlineCallbacks
def testForPPA(self):
"""Try to run publish-distro in PPA mode.
@@ -951,7 +1133,8 @@ class TestPublishDistroMethods(TestCaseWithFactory):
full_args = args + distro_args
script = PublishDistro(test_args=full_args)
script.distribution = distribution
- script.logger = DevNullLogger()
+ self.logger = BufferLogger()
+ script.logger = self.logger
return script
def test_isCareful_is_false_if_option_not_set(self):
@@ -1016,6 +1199,12 @@ class TestPublishDistroMethods(TestCaseWithFactory):
1, self.makeScript(args=["--copy-archive"]).countExclusiveOptions()
)
+ def test_countExclusiveOptions_counts_archive(self):
+ # countExclusiveOptions includes the "copy-archive" option.
+ self.assertEqual(
+ 1, self.makeScript(args=["--archive"]).countExclusiveOptions()
+ )
+
def test_countExclusiveOptions_detects_conflict(self):
# If more than one of the exclusive options has been set, that
# raises the result from countExclusiveOptions above 1.
@@ -1112,6 +1301,55 @@ class TestPublishDistroMethods(TestCaseWithFactory):
[], self.makeScript(all_derived=True).findDistros()
)
+ def test_findArchives_without_distro_filter(self):
+ ppa1 = self.factory.makeArchive(purpose=ArchivePurpose.PPA)
+ ppa2 = self.factory.makeArchive(purpose=ArchivePurpose.PPA)
+ non_existing_ppa_reference = "~foo/ubuntu/bar-ppa"
+
+ archvie_references = [
+ ppa1.reference,
+ ppa2.reference,
+ non_existing_ppa_reference,
+ ]
+ self.assertContentEqual(
+ [ppa1, ppa2],
+ self.makeScript(all_derived=True).findArchives(archvie_references),
+ )
+
+ self.assertIn(
+ "WARNING Cannot find the excluded archive with reference: '%s'"
+ % (non_existing_ppa_reference),
+ self.logger.getLogBuffer(),
+ )
+
+ def test_findArchives_with_distro_filter(self):
+ distro1 = self.makeDistro()
+ distro2 = self.makeDistro()
+ ppa1 = self.factory.makeArchive(distro1, purpose=ArchivePurpose.PPA)
+ ppa2 = self.factory.makeArchive(distro1, purpose=ArchivePurpose.PPA)
+ ppa3 = self.factory.makeArchive(distro2, purpose=ArchivePurpose.PPA)
+ non_existing_ppa_reference = "~foo/ubuntu/bar-ppa"
+
+ archvie_references = [
+ ppa1.reference,
+ ppa2.reference,
+ ppa3.reference,
+ non_existing_ppa_reference,
+ ]
+ self.assertContentEqual(
+ [ppa1, ppa2],
+ self.makeScript().findArchives(archvie_references, distro1),
+ )
+ self.assertContentEqual(
+ [ppa3], self.makeScript().findArchives(archvie_references, distro2)
+ )
+
+ self.assertIn(
+ "WARNING Cannot find the excluded archive with reference: '%s'"
+ % (non_existing_ppa_reference),
+ self.logger.getLogBuffer(),
+ )
+
def test_findSuite_finds_release_pocket(self):
# Despite its lack of a suffix, a release suite shows up
# normally in findSuite results.
@@ -1316,6 +1554,56 @@ class TestPublishDistroMethods(TestCaseWithFactory):
script = self.makeScript(distro, ["--ppa"])
self.assertContentEqual([], script.getTargetArchives(distro))
+ def test_getTargetArchives_ignores_excluded_archives_for_ppa(self):
+ # If the selected exclusive option is "ppa," getTargetArchives
+ # leaves out excluded PPAs.
+ distro = self.makeDistro()
+ ppa = self.factory.makeArchive(distro, purpose=ArchivePurpose.PPA)
+ excluded_ppa_1 = self.factory.makeArchive(
+ distro, purpose=ArchivePurpose.PPA
+ )
+ excluded_ppa_2 = self.factory.makeArchive(
+ distro, purpose=ArchivePurpose.PPA
+ )
+ script = self.makeScript(
+ distro,
+ [
+ "--ppa",
+ "--careful",
+ "--exclude",
+ excluded_ppa_1.reference,
+ "--exclude",
+ excluded_ppa_2.reference,
+ ],
+ )
+ self.assertContentEqual([ppa], script.getTargetArchives(distro))
+
+ def test_getTargetArchives_ignores_excluded_archives_for_private_ppa(self):
+ # If the selected exclusive option is "private-ppa," getTargetArchives
+ # leaves out excluded PPAs.
+ distro = self.makeDistro()
+ ppa = self.factory.makeArchive(
+ distro, purpose=ArchivePurpose.PPA, private=True
+ )
+ excluded_ppa_1 = self.factory.makeArchive(
+ distro, purpose=ArchivePurpose.PPA, private=True
+ )
+ excluded_ppa_2 = self.factory.makeArchive(
+ distro, purpose=ArchivePurpose.PPA, private=True
+ )
+ script = self.makeScript(
+ distro,
+ [
+ "--private-ppa",
+ "--careful",
+ "--exclude",
+ excluded_ppa_1.reference,
+ "--exclude",
+ excluded_ppa_2.reference,
+ ],
+ )
+ self.assertContentEqual([ppa], script.getTargetArchives(distro))
+
def test_getTargetArchives_gets_copy_archives(self):
# If the selected exclusive option is "copy-archive,"
# getTargetArchives looks for a copy archive.
@@ -1324,6 +1612,25 @@ class TestPublishDistroMethods(TestCaseWithFactory):
script = self.makeScript(distro, ["--copy-archive"])
self.assertContentEqual([copy], script.getTargetArchives(distro))
+ def test_getTargetArchives_gets_specific_archives(self):
+ # If the selected exclusive option is "archive,"
+ # getTargetArchives looks for the specified archives.
+ distro = self.makeDistro()
+
+ ppa_1 = self.factory.makeArchive(distro, purpose=ArchivePurpose.PPA)
+ ppa_2 = self.factory.makeArchive(distro, purpose=ArchivePurpose.PPA)
+
+ # create another random archive in the same distro
+ self.factory.makeArchive(distro, purpose=ArchivePurpose.PPA)
+
+ script = self.makeScript(
+ distro,
+ ["--archive", ppa_1.reference, "--archive", ppa_2.reference],
+ )
+ self.assertContentEqual(
+ [ppa_1, ppa_2], script.getTargetArchives(distro)
+ )
+
def test_getPublisher_returns_publisher(self):
# getPublisher produces a Publisher instance.
distro = self.makeDistro()
@@ -1806,3 +2113,11 @@ class TestPublishDistroMethods(TestCaseWithFactory):
self.assertTrue(by_hash_dir.is_dir())
# and still contains the two test files
self.assertEqual(2, len(list(by_hash_dir.iterdir())))
+
+ def test_lockfilename_option_overrides_default_lock(self):
+ script = self.makeScript(args=["--lockfilename", "foo.lock"])
+ assert script.lockfilepath == "/var/lock/foo.lock"
+
+ def test_default_lock(self):
+ script = self.makeScript()
+ assert script.lockfilepath == "/var/lock/%s" % GLOBAL_PUBLISHER_LOCK
Follow ups
