launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #24755
[Merge] ~cjwatson/launchpad:publish-by-null-datepublished into launchpad:master
Colin Watson has proposed merging ~cjwatson/launchpad:publish-by-null-datepublished into launchpad:master.
Commit message:
Publish based on NULL datepublished, not PENDING
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/384442
PackagePublishingStatus conflates the intended state of the publication (published, superseded, deleted, obsolete) with the current on-disk state (pending, published, etc.). This makes it difficult to add variations needed to support Built-Using.
Instead of publishing rows whose status is PENDING, publish ones that have an active status and a NULL value for datepublished, indicating that they have never been published to disk but that they are intended to be.
The PENDING status remains for API compatibility and as a hint in the web UI, but it no longer controls publishing.
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/launchpad:publish-by-null-datepublished into launchpad:master.
diff --git a/database/sampledata/current-dev.sql b/database/sampledata/current-dev.sql
index 191dd40..64cb226 100644
--- a/database/sampledata/current-dev.sql
+++ b/database/sampledata/current-dev.sql
@@ -4523,7 +4523,7 @@ INSERT INTO sourcepackagepublishinghistory (id, sourcepackagerelease, distroseri
INSERT INTO sourcepackagepublishinghistory (id, sourcepackagerelease, distroseries, status, component, section, datecreated, datepublished, datesuperseded, supersededby, datemadepending, scheduleddeletiondate, dateremoved, pocket, archive, removed_by, removal_comment, ancestor, sourcepackagename, creator, sponsor, packageupload) VALUES (27, 33, 10, 2, 1, 3, '2007-07-09 13:44:00.10654', '2007-07-09 14:14:00', NULL, NULL, NULL, NULL, NULL, 0, 9, NULL, NULL, NULL, 24, NULL, NULL, NULL);
INSERT INTO sourcepackagepublishinghistory (id, sourcepackagerelease, distroseries, status, component, section, datecreated, datepublished, datesuperseded, supersededby, datemadepending, scheduleddeletiondate, dateremoved, pocket, archive, removed_by, removal_comment, ancestor, sourcepackagename, creator, sponsor, packageupload) VALUES (28, 35, 1, 2, 1, 3, '2007-07-09 12:00:00', '2007-07-09 12:00:01', NULL, NULL, NULL, NULL, NULL, 0, 9, NULL, NULL, NULL, 26, NULL, NULL, NULL);
INSERT INTO sourcepackagepublishinghistory (id, sourcepackagerelease, distroseries, status, component, section, datecreated, datepublished, datesuperseded, supersededby, datemadepending, scheduleddeletiondate, dateremoved, pocket, archive, removed_by, removal_comment, ancestor, sourcepackagename, creator, sponsor, packageupload) VALUES (29, 20, 1, 2, 1, 3, '2007-07-09 12:00:00', '2007-07-09 12:00:01', NULL, NULL, NULL, NULL, NULL, 0, 9, NULL, NULL, NULL, 14, NULL, NULL, NULL);
-INSERT INTO sourcepackagepublishinghistory (id, sourcepackagerelease, distroseries, status, component, section, datecreated, datepublished, datesuperseded, supersededby, datemadepending, scheduleddeletiondate, dateremoved, pocket, archive, removed_by, removal_comment, ancestor, sourcepackagename, creator, sponsor, packageupload) VALUES (30, 36, 10, 2, 5, 7, '2007-08-09 21:28:12.106187', NULL, NULL, NULL, NULL, NULL, NULL, 0, 12, NULL, NULL, NULL, 27, NULL, NULL, NULL);
+INSERT INTO sourcepackagepublishinghistory (id, sourcepackagerelease, distroseries, status, component, section, datecreated, datepublished, datesuperseded, supersededby, datemadepending, scheduleddeletiondate, dateremoved, pocket, archive, removed_by, removal_comment, ancestor, sourcepackagename, creator, sponsor, packageupload) VALUES (30, 36, 10, 2, 5, 7, '2007-08-09 21:28:12.106187', '2007-08-09 21:28:13', NULL, NULL, NULL, NULL, NULL, 0, 12, NULL, NULL, NULL, 27, NULL, NULL, NULL);
INSERT INTO sourcepackagepublishinghistory (id, sourcepackagerelease, distroseries, status, component, section, datecreated, datepublished, datesuperseded, supersededby, datemadepending, scheduleddeletiondate, dateremoved, pocket, archive, removed_by, removal_comment, ancestor, sourcepackagename, creator, sponsor, packageupload) VALUES (31, 35, 10, 2, 1, 3, '2007-07-09 12:00:00', '2007-07-09 12:00:01', NULL, NULL, NULL, NULL, NULL, 0, 10, NULL, NULL, NULL, 26, NULL, NULL, NULL);
INSERT INTO sourcepackagepublishinghistory (id, sourcepackagerelease, distroseries, status, component, section, datecreated, datepublished, datesuperseded, supersededby, datemadepending, scheduleddeletiondate, dateremoved, pocket, archive, removed_by, removal_comment, ancestor, sourcepackagename, creator, sponsor, packageupload) VALUES (32, 38, 14, 1, 1, 3, '2006-02-15 12:19:00.05741', NULL, NULL, NULL, NULL, NULL, NULL, 0, 16, NULL, NULL, NULL, 19, NULL, NULL, NULL);
INSERT INTO sourcepackagepublishinghistory (id, sourcepackagerelease, distroseries, status, component, section, datecreated, datepublished, datesuperseded, supersededby, datemadepending, scheduleddeletiondate, dateremoved, pocket, archive, removed_by, removal_comment, ancestor, sourcepackagename, creator, sponsor, packageupload) VALUES (33, 39, 1, 1, 1, 3, '2006-02-15 12:19:00.05741', NULL, NULL, NULL, NULL, NULL, NULL, 0, 1, NULL, NULL, NULL, 9, NULL, NULL, NULL);
diff --git a/database/sampledata/current.sql b/database/sampledata/current.sql
index be42368..5e9f4ea 100644
--- a/database/sampledata/current.sql
+++ b/database/sampledata/current.sql
@@ -4447,7 +4447,7 @@ INSERT INTO sourcepackagepublishinghistory (id, sourcepackagerelease, distroseri
INSERT INTO sourcepackagepublishinghistory (id, sourcepackagerelease, distroseries, status, component, section, datecreated, datepublished, datesuperseded, supersededby, datemadepending, scheduleddeletiondate, dateremoved, pocket, archive, removed_by, removal_comment, ancestor, sourcepackagename, creator, sponsor, packageupload) VALUES (27, 33, 10, 2, 1, 3, '2007-07-09 13:44:00.10654', '2007-07-09 14:14:00', NULL, NULL, NULL, NULL, NULL, 0, 9, NULL, NULL, NULL, 24, NULL, NULL, NULL);
INSERT INTO sourcepackagepublishinghistory (id, sourcepackagerelease, distroseries, status, component, section, datecreated, datepublished, datesuperseded, supersededby, datemadepending, scheduleddeletiondate, dateremoved, pocket, archive, removed_by, removal_comment, ancestor, sourcepackagename, creator, sponsor, packageupload) VALUES (28, 35, 1, 2, 1, 3, '2007-07-09 12:00:00', '2007-07-09 12:00:01', NULL, NULL, NULL, NULL, NULL, 0, 9, NULL, NULL, NULL, 26, NULL, NULL, NULL);
INSERT INTO sourcepackagepublishinghistory (id, sourcepackagerelease, distroseries, status, component, section, datecreated, datepublished, datesuperseded, supersededby, datemadepending, scheduleddeletiondate, dateremoved, pocket, archive, removed_by, removal_comment, ancestor, sourcepackagename, creator, sponsor, packageupload) VALUES (29, 20, 1, 2, 1, 3, '2007-07-09 12:00:00', '2007-07-09 12:00:01', NULL, NULL, NULL, NULL, NULL, 0, 9, NULL, NULL, NULL, 14, NULL, NULL, NULL);
-INSERT INTO sourcepackagepublishinghistory (id, sourcepackagerelease, distroseries, status, component, section, datecreated, datepublished, datesuperseded, supersededby, datemadepending, scheduleddeletiondate, dateremoved, pocket, archive, removed_by, removal_comment, ancestor, sourcepackagename, creator, sponsor, packageupload) VALUES (30, 36, 10, 2, 5, 7, '2007-08-09 21:28:12.106187', NULL, NULL, NULL, NULL, NULL, NULL, 0, 12, NULL, NULL, NULL, 27, NULL, NULL, NULL);
+INSERT INTO sourcepackagepublishinghistory (id, sourcepackagerelease, distroseries, status, component, section, datecreated, datepublished, datesuperseded, supersededby, datemadepending, scheduleddeletiondate, dateremoved, pocket, archive, removed_by, removal_comment, ancestor, sourcepackagename, creator, sponsor, packageupload) VALUES (30, 36, 10, 2, 5, 7, '2007-08-09 21:28:12.106187', '2007-08-09 21:28:13', NULL, NULL, NULL, NULL, NULL, 0, 12, NULL, NULL, NULL, 27, NULL, NULL, NULL);
INSERT INTO sourcepackagepublishinghistory (id, sourcepackagerelease, distroseries, status, component, section, datecreated, datepublished, datesuperseded, supersededby, datemadepending, scheduleddeletiondate, dateremoved, pocket, archive, removed_by, removal_comment, ancestor, sourcepackagename, creator, sponsor, packageupload) VALUES (31, 35, 10, 2, 1, 3, '2007-07-09 12:00:00', '2007-07-09 12:00:01', NULL, NULL, NULL, NULL, NULL, 0, 10, NULL, NULL, NULL, 26, NULL, NULL, NULL);
diff --git a/lib/lp/archivepublisher/publishing.py b/lib/lp/archivepublisher/publishing.py
index f6246b0..18f273a 100644
--- a/lib/lp/archivepublisher/publishing.py
+++ b/lib/lp/archivepublisher/publishing.py
@@ -537,16 +537,21 @@ class Publisher(object):
def getPendingSourcePublications(self, is_careful):
"""Return the specific group of source records to be published."""
- # Careful publishing should include all PUBLISHED rows, normal run
- # only includes PENDING ones.
- statuses = [PackagePublishingStatus.PENDING]
- if is_careful:
- statuses.append(PackagePublishingStatus.PUBLISHED)
+ # Careful publishing should include all rows in active statuses
+ # regardless of whether they have previously been published; a
+ # normal run only includes rows in active statuses that have never
+ # been published.
+ clauses = [
+ SourcePackagePublishingHistory.archive == self.archive,
+ SourcePackagePublishingHistory.status.is_in(
+ active_publishing_status),
+ ]
+ if not is_careful:
+ clauses.append(
+ SourcePackagePublishingHistory.datepublished == None)
publications = IStore(SourcePackagePublishingHistory).find(
- SourcePackagePublishingHistory,
- SourcePackagePublishingHistory.archive == self.archive,
- SourcePackagePublishingHistory.status.is_in(statuses))
+ SourcePackagePublishingHistory, *clauses)
return publications.order_by(
SourcePackagePublishingHistory.distroseriesID,
SourcePackagePublishingHistory.pocket,
@@ -588,19 +593,21 @@ class Publisher(object):
def getPendingBinaryPublications(self, is_careful):
"""Return the specific group of binary records to be published."""
- statuses = [PackagePublishingStatus.PENDING]
- if is_careful:
- statuses.append(PackagePublishingStatus.PUBLISHED)
-
- publications = IStore(BinaryPackagePublishingHistory).find(
- BinaryPackagePublishingHistory,
+ clauses = [
BinaryPackagePublishingHistory.archive == self.archive,
BinaryPackagePublishingHistory.distroarchseriesID ==
DistroArchSeries.id,
- DistroArchSeries.distroseriesID == DistroSeries.id,
- BinaryPackagePublishingHistory.status.is_in(statuses))
+ BinaryPackagePublishingHistory.status.is_in(
+ active_publishing_status),
+ ]
+ if not is_careful:
+ clauses.append(
+ BinaryPackagePublishingHistory.datepublished == None)
+
+ publications = IStore(BinaryPackagePublishingHistory).find(
+ BinaryPackagePublishingHistory, *clauses)
return publications.order_by(
- DistroSeries.id,
+ DistroArchSeries.distroseriesID,
BinaryPackagePublishingHistory.pocket,
DistroArchSeries.architecturetag,
Desc(BinaryPackagePublishingHistory.id))
diff --git a/lib/lp/archivepublisher/scripts/publish_ftpmaster.py b/lib/lp/archivepublisher/scripts/publish_ftpmaster.py
index c307846..2610531 100644
--- a/lib/lp/archivepublisher/scripts/publish_ftpmaster.py
+++ b/lib/lp/archivepublisher/scripts/publish_ftpmaster.py
@@ -43,10 +43,7 @@ from lp.services.scripts.base import (
LaunchpadScriptFailure,
)
from lp.services.utils import file_exists
-from lp.soyuz.enums import (
- ArchivePurpose,
- PackagePublishingStatus,
- )
+from lp.soyuz.enums import ArchivePurpose
from lp.soyuz.model.distroarchseries import DistroArchSeries
from lp.soyuz.scripts.custom_uploads_copier import CustomUploadsCopier
@@ -265,10 +262,10 @@ class PublishFTPMaster(LaunchpadCronScript):
self.logger.debug("Querying which suites are pending publication...")
archive = distribution.main_archive
- pending = PackagePublishingStatus.PENDING
- pending_sources = list(archive.getPublishedSources(status=pending))
+ pending_sources = list(archive.getPublishedSources(
+ only_unpublished=True))
pending_binaries = list(archive.getAllPublishedBinaries(
- status=pending))
+ only_unpublished=True))
load_related(
DistroArchSeries, pending_binaries, ['distroarchseriesID'])
return set(
diff --git a/lib/lp/registry/model/distribution.py b/lib/lp/registry/model/distribution.py
index 9c4c96e..33c1571 100644
--- a/lib/lp/registry/model/distribution.py
+++ b/lib/lp/registry/model/distribution.py
@@ -176,6 +176,7 @@ from lp.soyuz.interfaces.binarypackagebuild import IBinaryPackageBuildSet
from lp.soyuz.interfaces.buildrecords import IHasBuildRecords
from lp.soyuz.interfaces.publishing import active_publishing_status
from lp.soyuz.model.archive import Archive
+from lp.soyuz.model.archivefile import ArchiveFile
from lp.soyuz.model.binarypackagename import BinaryPackageName
from lp.soyuz.model.distributionsourcepackagerelease import (
DistributionSourcePackageRelease,
@@ -1288,60 +1289,59 @@ class Distribution(SQLBase, BugTargetBase, MakesAnnouncements,
def getPendingPublicationPPAs(self):
"""See `IDistribution`."""
- src_query = """
- Archive.purpose = %s AND
- Archive.distribution = %s AND
- SourcePackagePublishingHistory.archive = archive.id AND
- SourcePackagePublishingHistory.scheduleddeletiondate IS NULL AND
- SourcePackagePublishingHistory.dateremoved IS NULL AND
- SourcePackagePublishingHistory.status IN (%s, %s)
- """ % sqlvalues(ArchivePurpose.PPA, self,
- PackagePublishingStatus.PENDING,
- PackagePublishingStatus.DELETED)
-
- src_archives = Archive.select(
- src_query, clauseTables=['SourcePackagePublishingHistory'],
- orderBy=['archive.id'], distinct=True)
-
- bin_query = """
- Archive.purpose = %s AND
- Archive.distribution = %s AND
- BinaryPackagePublishingHistory.archive = archive.id AND
- BinaryPackagePublishingHistory.scheduleddeletiondate IS NULL AND
- BinaryPackagePublishingHistory.dateremoved IS NULL AND
- BinaryPackagePublishingHistory.status IN (%s, %s)
- """ % sqlvalues(ArchivePurpose.PPA, self,
- PackagePublishingStatus.PENDING,
- PackagePublishingStatus.DELETED)
-
- bin_archives = Archive.select(
- bin_query, clauseTables=['BinaryPackagePublishingHistory'],
- orderBy=['archive.id'], distinct=True)
-
- reapable_af_query = """
- Archive.purpose = %s AND
- Archive.distribution = %s AND
- ArchiveFile.archive = archive.id AND
- ArchiveFile.scheduled_deletion_date < %s
- """ % sqlvalues(ArchivePurpose.PPA, self, UTC_NOW)
-
- reapable_af_archives = Archive.select(
- reapable_af_query, clauseTables=['ArchiveFile'],
- orderBy=['archive.id'], distinct=True)
-
- dirty_suites_query = """
- Archive.purpose = %s AND
- Archive.distribution = %s AND
- Archive.dirty_suites IS NOT NULL
- """ % sqlvalues(ArchivePurpose.PPA, self)
+ src_archives = IStore(Archive).find(
+ Archive,
+ Archive.purpose == ArchivePurpose.PPA,
+ Archive.distribution == self,
+ SourcePackagePublishingHistory.archive == Archive.id,
+ SourcePackagePublishingHistory.scheduleddeletiondate == None,
+ SourcePackagePublishingHistory.dateremoved == None,
+ Or(
+ And(
+ SourcePackagePublishingHistory.status.is_in(
+ active_publishing_status),
+ SourcePackagePublishingHistory.datepublished == None),
+ SourcePackagePublishingHistory.status ==
+ PackagePublishingStatus.DELETED,
+ )).order_by(Archive.id).config(distinct=True)
+
+ bin_archives = IStore(Archive).find(
+ Archive,
+ Archive.purpose == ArchivePurpose.PPA,
+ Archive.distribution == self,
+ BinaryPackagePublishingHistory.archive == Archive.id,
+ BinaryPackagePublishingHistory.scheduleddeletiondate == None,
+ BinaryPackagePublishingHistory.dateremoved == None,
+ Or(
+ And(
+ BinaryPackagePublishingHistory.status.is_in(
+ active_publishing_status),
+ BinaryPackagePublishingHistory.datepublished == None),
+ BinaryPackagePublishingHistory.status ==
+ PackagePublishingStatus.DELETED,
+ )).order_by(Archive.id).config(distinct=True)
+
+ reapable_af_archives = IStore(Archive).find(
+ Archive,
+ Archive.purpose == ArchivePurpose.PPA,
+ Archive.distribution == self,
+ ArchiveFile.archive == Archive.id,
+ ArchiveFile.scheduled_deletion_date < UTC_NOW,
+ ).order_by(Archive.id).config(distinct=True)
- dirty_suites_archives = Archive.select(
- dirty_suites_query, orderBy=['archive.id'])
+ dirty_suites_archives = IStore(Archive).find(
+ Archive,
+ Archive.purpose == ArchivePurpose.PPA,
+ Archive.distribution == self,
+ Archive.dirty_suites != None,
+ ).order_by(Archive.id)
- deleting_archives = Archive.selectBy(
- distribution=self,
- purpose=ArchivePurpose.PPA,
- status=ArchiveStatus.DELETING).orderBy(['archive.id'])
+ deleting_archives = IStore(Archive).find(
+ Archive,
+ Archive.purpose == ArchivePurpose.PPA,
+ Archive.distribution == self,
+ Archive.status == ArchiveStatus.DELETING,
+ ).order_by(Archive.id)
return src_archives.union(bin_archives).union(
reapable_af_archives).union(dirty_suites_archives).union(
diff --git a/lib/lp/soyuz/interfaces/archive.py b/lib/lp/soyuz/interfaces/archive.py
index 6347193..191da83 100644
--- a/lib/lp/soyuz/interfaces/archive.py
+++ b/lib/lp/soyuz/interfaces/archive.py
@@ -537,7 +537,8 @@ class IArchiveSubscriberView(Interface):
distroseries=None, pocket=None,
exact_match=False, created_since_date=None,
eager_load=False, component_name=None,
- order_by_date=False):
+ order_by_date=False, include_removed=True,
+ only_unpublished=False):
"""All `ISourcePackagePublishingHistory` target to this archive.
:param name: source name filter (exact match or SQL LIKE controlled
@@ -560,6 +561,10 @@ class IArchiveSubscriberView(Interface):
If not specified, publications are ordered by source
package name (lexicographically), then by descending version
and then descending ID.
+ :param include_removed: If True, include publications that have been
+ removed from disk as well as those that have not.
+ :param only_unpublished: If True, only include publications that
+ have never been published to disk.
:return: SelectResults containing `ISourcePackagePublishingHistory`,
ordered by name. If there are multiple results for the same
@@ -1240,6 +1245,7 @@ class IArchiveView(IHasBuildRecords):
distroarchseries=None, pocket=None,
exact_match=False, created_since_date=None,
ordered=True, order_by_date=False,
+ include_removed=True, only_unpublished=False,
eager_load=False):
"""All `IBinaryPackagePublishingHistory` target to this archive.
@@ -1261,6 +1267,10 @@ class IArchiveView(IHasBuildRecords):
:param order_by_date: Order publications by descending creation date
and then by descending ID. This is suitable for applications
that need to catch up with publications since their last run.
+ :param include_removed: If True, include publications that have been
+ removed from disk as well as those that have not.
+ :param only_unpublished: If True, only include publications that
+ have never been published to disk.
:return: A collection containing `BinaryPackagePublishingHistory`.
"""
diff --git a/lib/lp/soyuz/model/archive.py b/lib/lp/soyuz/model/archive.py
index c496824..acf8541 100644
--- a/lib/lp/soyuz/model/archive.py
+++ b/lib/lp/soyuz/model/archive.py
@@ -617,7 +617,8 @@ class Archive(SQLBase):
distroseries=None, pocket=None,
exact_match=False, created_since_date=None,
eager_load=False, component_name=None,
- order_by_date=False, include_removed=True):
+ order_by_date=False, include_removed=True,
+ only_unpublished=False):
"""See `IArchive`."""
clauses = [SourcePackagePublishingHistory.archiveID == self.id]
@@ -693,6 +694,10 @@ class Archive(SQLBase):
if not include_removed:
clauses.append(SourcePackagePublishingHistory.dateremoved == None)
+ if only_unpublished:
+ clauses.append(
+ SourcePackagePublishingHistory.datepublished == None)
+
store = Store.of(self)
resultset = store.find(
SourcePackagePublishingHistory, *clauses).order_by(*order_by)
@@ -803,7 +808,7 @@ class Archive(SQLBase):
self, name=None, version=None, status=None, distroarchseries=None,
pocket=None, exact_match=False, created_since_date=None,
ordered=True, order_by_date=False, include_removed=True,
- need_bpr=False):
+ only_unpublished=False, need_bpr=False):
"""Base clauses for binary publishing queries.
Returns a list of 'clauses' (to be joined in the callsite).
@@ -880,13 +885,18 @@ class Archive(SQLBase):
if not include_removed:
clauses.append(BinaryPackagePublishingHistory.dateremoved == None)
+ if only_unpublished:
+ clauses.append(
+ BinaryPackagePublishingHistory.datepublished == None)
+
return clauses, order_by
def getAllPublishedBinaries(self, name=None, version=None, status=None,
distroarchseries=None, pocket=None,
exact_match=False, created_since_date=None,
ordered=True, order_by_date=False,
- include_removed=True, eager_load=False):
+ include_removed=True, only_unpublished=False,
+ eager_load=False):
"""See `IArchive`."""
# Circular imports.
from lp.registry.model.distroseries import DistroSeries
@@ -896,7 +906,8 @@ class Archive(SQLBase):
name=name, version=version, status=status, pocket=pocket,
distroarchseries=distroarchseries, exact_match=exact_match,
created_since_date=created_since_date, ordered=ordered,
- order_by_date=order_by_date, include_removed=include_removed)
+ order_by_date=order_by_date, include_removed=include_removed,
+ only_unpublished=only_unpublished)
result = Store.of(self).find(
BinaryPackagePublishingHistory, *clauses).order_by(*order_by)
diff --git a/lib/lp/soyuz/model/publishing.py b/lib/lp/soyuz/model/publishing.py
index 1c5d384..4e5b77a 100644
--- a/lib/lp/soyuz/model/publishing.py
+++ b/lib/lp/soyuz/model/publishing.py
@@ -157,7 +157,8 @@ class ArchivePublisherBase:
"""see IArchiveSafePublisher."""
# XXX cprov 2006-06-14:
# Implement sanity checks before set it as published
- if self.status == PackagePublishingStatus.PENDING:
+ if (self.status in active_publishing_status and
+ self.datepublished is None):
# update the DB publishing record status if they
# are pending, don't do anything for the ones
# already published (usually when we use -C
