launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #20510
[Merge] lp:~cjwatson/launchpad/sspb-cache into lp:launchpad
Colin Watson has proposed merging lp:~cjwatson/launchpad/sspb-cache into lp:launchpad with lp:~cjwatson/launchpad/update-pkgcache-on-publish as a prerequisite.
Commit message:
Cache SeriesSourcePackageBranch rows in DistributionSourcePackageCache.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/sspb-cache/+merge/295852
Cache SeriesSourcePackageBranch rows in DistributionSourcePackageCache. This is the next step towards being able to make various vocabularies cache- and archive-aware rather than doing all the heavy lifting themselves.
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/sspb-cache into lp:launchpad.
=== modified file 'cronscripts/update-pkgcache.py'
--- cronscripts/update-pkgcache.py 2013-01-07 02:40:55 +0000
+++ cronscripts/update-pkgcache.py 2016-05-26 16:31:14 +0000
@@ -40,7 +40,8 @@
"""Update package caches for the given location.
'archive' can be one of the main archives (PRIMARY, PARTNER or
- EMBARGOED) or even a PPA.
+ EMBARGOED), a PPA, or None to update caches of official branch
+ links.
This method commits the transaction frequently since it deal with
a huge amount of data.
@@ -48,8 +49,9 @@
PPA archives caches are consolidated in a Archive row to optimize
searches across PPAs.
"""
- for distroseries in distribution.series:
- self.updateDistroSeriesCache(distroseries, archive)
+ if archive is not None:
+ for distroseries in distribution.series:
+ self.updateDistroSeriesCache(distroseries, archive)
DistributionSourcePackageCache.removeOld(
distribution, archive, log=self.logger)
@@ -91,6 +93,10 @@
self.updateDistributionCache(distribution, archive)
self.logger.info(
+ 'Updating %s official branch links' % distribution.name)
+ self.updateDistributionCache(distribution, None)
+
+ self.logger.info(
'Updating %s PPAs' % distribution.name)
for archive in distribution.getAllPPAs():
self.updateDistributionCache(distribution, archive)
=== modified file 'database/schema/security.cfg'
--- database/schema/security.cfg 2016-05-26 16:31:14 +0000
+++ database/schema/security.cfg 2016-05-26 16:31:14 +0000
@@ -386,6 +386,7 @@
public.product = SELECT
public.productseries = SELECT
public.question = SELECT
+public.seriessourcepackagebranch = SELECT
public.sourcepackagename = SELECT
public.sourcepackagepublishinghistory = SELECT
public.sourcepackagerelease = SELECT
=== modified file 'lib/lp/code/model/seriessourcepackagebranch.py'
--- lib/lp/code/model/seriessourcepackagebranch.py 2015-07-08 16:05:11 +0000
+++ lib/lp/code/model/seriessourcepackagebranch.py 2016-05-26 16:31:14 +0000
@@ -83,12 +83,19 @@
def new(distroseries, pocket, sourcepackagename, branch, registrant,
date_created=None):
"""Link a source package in a distribution suite to a branch."""
+ # Circular import.
+ from lp.soyuz.model.distributionsourcepackagecache import (
+ DistributionSourcePackageCache,
+ )
+
if date_created is None:
date_created = datetime.now(pytz.UTC)
sspb = SeriesSourcePackageBranch(
distroseries, pocket, sourcepackagename, branch, registrant,
date_created)
IMasterStore(SeriesSourcePackageBranch).add(sspb)
+ DistributionSourcePackageCache.updateOfficialBranches(
+ distroseries.distribution, [sourcepackagename])
return sspb
def findForBranch(self, branch):
=== modified file 'lib/lp/registry/model/distribution.py'
--- lib/lp/registry/model/distribution.py 2016-03-22 02:43:37 +0000
+++ lib/lp/registry/model/distribution.py 2016-05-26 16:31:14 +0000
@@ -1005,8 +1005,10 @@
conditions = [
DistributionSourcePackageCache.distribution == self,
- DistributionSourcePackageCache.archiveID.is_in(
- self.all_distro_archive_ids),
+ Or(
+ DistributionSourcePackageCache.archiveID.is_in(
+ self.all_distro_archive_ids),
+ DistributionSourcePackageCache.archive == None),
Or(
fti_search(DistributionSourcePackageCache, text),
DistributionSourcePackageCache.name.contains_string(
=== modified file 'lib/lp/soyuz/doc/package-cache-script.txt'
--- lib/lp/soyuz/doc/package-cache-script.txt 2016-05-24 16:52:57 +0000
+++ lib/lp/soyuz/doc/package-cache-script.txt 2016-05-26 16:31:14 +0000
@@ -50,6 +50,7 @@
INFO Updating ubuntu package counters
INFO Updating ubuntu main archives
...
+ INFO Updating ubuntu official branch links
INFO Updating ubuntu PPAs
...
INFO redhat done
=== modified file 'lib/lp/soyuz/doc/package-cache.txt'
--- lib/lp/soyuz/doc/package-cache.txt 2016-05-18 08:28:54 +0000
+++ lib/lp/soyuz/doc/package-cache.txt 2016-05-26 16:31:14 +0000
@@ -451,6 +451,56 @@
>>> cprov.archive.updateArchiveCache()
+Official branch caches
+======================
+
+The source package names referred to by official branch links are cached
+with a null archive column.
+
+ >>> from lp.registry.interfaces.pocket import PackagePublishingPocket
+ >>> from lp.testing.dbuser import lp_dbuser
+
+ >>> with lp_dbuser():
+ ... branch = factory.makePackageBranch(
+ ... distroseries=ubuntu.currentseries)
+ >>> ubuntu.searchSourcePackages(branch.sourcepackagename.name).count()
+ 0
+ >>> with lp_dbuser():
+ ... branch.sourcepackage.setBranch(
+ ... PackagePublishingPocket.RELEASE, branch, branch.owner)
+ >>> ubuntu.searchSourcePackages(branch.sourcepackagename.name).count()
+ 1
+
+Updating the cache adds missing entries to the cache.
+
+ >>> branch_cache = DistributionSourcePackageCache.selectOneBy(
+ ... archive=None, distribution=ubuntu,
+ ... name=branch.sourcepackagename.name)
+ >>> removeSecurityProxy(branch_cache).destroySelf()
+ >>> ubuntu.searchSourcePackages(branch.sourcepackagename.name).count()
+ 0
+ >>> updates = DistributionSourcePackageCache.updateAll(
+ ... ubuntu, archive=None, ztm=transaction, log=FakeLogger(),
+ ... commit_chunk=3)
+ DEBUG Considering sources unique-from-factory-...
+ ...
+ >>> print updates
+ 1
+ >>> ubuntu.searchSourcePackages(branch.sourcepackagename.name).count()
+ 1
+
+After removing the link, the removeOld method will remove the cache entry.
+
+ >>> with lp_dbuser():
+ ... branch.sourcepackage.setBranch(
+ ... PackagePublishingPocket.RELEASE, None, branch.owner)
+ >>> DistributionSourcePackageCache.removeOld(
+ ... ubuntu, archive=None, log=FakeLogger())
+ DEBUG Removing source cache for 'unique-from-factory...' (...)
+ >>> ubuntu.searchSourcePackages(branch.sourcepackagename.name).count()
+ 0
+
+
Package Counters
================
@@ -483,10 +533,7 @@
>>> from lp.soyuz.enums import PackagePublishingStatus
>>> from lp.soyuz.tests.test_publishing import SoyuzTestPublisher
- >>> from lp.testing.dbuser import (
- ... lp_dbuser,
- ... switch_dbuser,
- ... )
+ >>> from lp.testing.dbuser import switch_dbuser
>>> test_publisher = SoyuzTestPublisher()
@@ -757,5 +804,3 @@
>>> print_search_results(u'cprov')
PPA for Celso Providelo
-
-
=== modified file 'lib/lp/soyuz/model/distributionsourcepackagecache.py'
--- lib/lp/soyuz/model/distributionsourcepackagecache.py 2016-05-26 16:31:14 +0000
+++ lib/lp/soyuz/model/distributionsourcepackagecache.py 2016-05-26 16:31:14 +0000
@@ -16,6 +16,8 @@
)
from zope.interface import implementer
+from lp.code.model.seriessourcepackagebranch import SeriesSourcePackageBranch
+from lp.registry.model.distroseries import DistroSeries
from lp.registry.model.sourcepackagename import SourcePackageName
from lp.services.database import bulk
from lp.services.database.decoratedresultset import DecoratedResultSet
@@ -32,12 +34,15 @@
from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
+_DEFAULT = object()
+
+
@implementer(IDistributionSourcePackageCache)
class DistributionSourcePackageCache(SQLBase):
_table = 'DistributionSourcePackageCache'
archive = ForeignKey(dbName='archive',
- foreignKey='Archive', notNull=True)
+ foreignKey='Archive', notNull=False)
distribution = ForeignKey(dbName='distribution',
foreignKey='Distribution', notNull=True)
sourcepackagename = ForeignKey(dbName='sourcepackagename',
@@ -61,32 +66,41 @@
self.sourcepackagename)
@classmethod
- def findCurrentSourcePackageNames(cls, archive):
- spn_ids = IStore(SourcePackagePublishingHistory).find(
- SourcePackagePublishingHistory.sourcepackagenameID,
- SourcePackagePublishingHistory.archive == archive,
- SourcePackagePublishingHistory.status.is_in((
- PackagePublishingStatus.PENDING,
- PackagePublishingStatus.PUBLISHED))).config(
- distinct=True)
- return bulk.load(SourcePackageName, spn_ids)
+ def findCurrentSourcePackageNames(cls, distro, archive):
+ if archive is None:
+ spn_ids = IStore(SeriesSourcePackageBranch).find(
+ SeriesSourcePackageBranch.sourcepackagenameID,
+ DistroSeries.distribution == distro.id,
+ SeriesSourcePackageBranch.distroseriesID == DistroSeries.id)
+ else:
+ spn_ids = IStore(SourcePackagePublishingHistory).find(
+ SourcePackagePublishingHistory.sourcepackagenameID,
+ SourcePackagePublishingHistory.archive == archive,
+ SourcePackagePublishingHistory.status.is_in((
+ PackagePublishingStatus.PENDING,
+ PackagePublishingStatus.PUBLISHED)))
+ return bulk.load(SourcePackageName, spn_ids.config(distinct=True))
@classmethod
- def _find(cls, distro, archive=None):
+ def _find(cls, distro, archive=_DEFAULT):
"""The set of all source package info caches for this distribution.
If 'archive' is not given it will return all caches stored for the
distribution main archives (PRIMARY and PARTNER).
"""
- if archive is not None:
- archives = [archive.id]
+ archive_column = DistributionSourcePackageCache.archiveID
+ if archive is _DEFAULT:
+ archive_clause = (
+ archive_column.is_in(distro.all_distro_archive_ids))
+ elif archive is not None:
+ archive_clause = (archive_column == archive.id)
else:
- archives = distro.all_distro_archive_ids
+ archive_clause = (archive_column == None)
result = IStore(DistributionSourcePackageCache).find(
(DistributionSourcePackageCache, SourcePackageName),
DistributionSourcePackageCache.distribution == distro,
- DistributionSourcePackageCache.archiveID.is_in(archives),
+ archive_clause,
SourcePackageName.id ==
DistributionSourcePackageCache.sourcepackagenameID,
).order_by(DistributionSourcePackageCache.name)
@@ -98,17 +112,18 @@
Also purges all existing cache records for disabled archives.
- :param archive: target `IArchive`.
+ :param archive: target `IArchive`, or None to consider official
+ branches.
:param log: the context logger object able to print DEBUG level
messages.
"""
# Get the set of source package names to deal with.
- if not archive.enabled:
+ if archive is not None and not archive.enabled:
spns = set()
else:
spns = set(
- cls.findCurrentSourcePackageNames(archive))
+ cls.findCurrentSourcePackageNames(distro, archive))
# Remove the cache entries for packages we no longer publish.
for cache in cls._find(distro, archive):
@@ -155,7 +170,7 @@
[spn.id for spn in sourcepackagenames]))
cache_map = {cache.sourcepackagename: cache for cache in all_caches}
- for spn in set(sourcepackagenames) - set(cache_map.keys()):
+ for spn in set(sourcepackagenames) - set(cache_map):
cache_map[spn] = cls(
archive=archive, distribution=distro,
sourcepackagename=spn)
@@ -217,6 +232,23 @@
cache.changelog = None
@classmethod
+ def updateOfficialBranches(cls, distro, sourcepackagenames):
+ """Update the package cache for official branches with given names.
+
+ We just cache the names for these.
+ """
+ all_caches = IStore(cls).find(
+ cls, cls.distribution == distro, cls.archive == None,
+ cls.sourcepackagenameID.is_in(
+ [spn.id for spn in sourcepackagenames]))
+ cache_map = {cache.sourcepackagename: cache for cache in all_caches}
+
+ for spn in set(sourcepackagenames) - set(cache_map):
+ cache_map[spn] = cls(
+ archive=None, distribution=distro, sourcepackagename=spn,
+ name=spn.name)
+
+ @classmethod
def updateAll(cls, distro, archive, log, ztm, commit_chunk=500):
"""Update the source package cache.
@@ -232,12 +264,12 @@
:return the number packages updated done
"""
# Do not create cache entries for disabled archives.
- if not archive.enabled:
+ if archive is not None and not archive.enabled:
return
# Get the set of source package names to deal with.
spns = list(sorted(
- cls.findCurrentSourcePackageNames(archive),
+ cls.findCurrentSourcePackageNames(distro, archive),
key=attrgetter('name')))
number_of_updates = 0
@@ -255,7 +287,10 @@
log.debug(
"Considering sources %s",
', '.join([spn.name for spn in chunk]))
- cls.update(distro, chunk, archive, log)
+ if archive is None:
+ cls.updateOfficialBranches(distro, chunk)
+ else:
+ cls.update(distro, chunk, archive, log)
number_of_updates += len(chunk)
log.debug("Committing")
ztm.commit()
Follow ups
