launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #20059
[Merge] lp:~wgrant/launchpad/gpgkey-fks into lp:launchpad
William Grant has proposed merging lp:~wgrant/launchpad/gpgkey-fks into lp:launchpad with lp:~wgrant/launchpad/gpgkey-fks-db as a prerequisite.
Commit message:
Set GPGKey FK replacement columns and add garbo jobs to backfill.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~wgrant/launchpad/gpgkey-fks/+merge/287710
Set GPGKey FK replacement columns and add garbo jobs to backfill.
Using a Storm validator would be lovely, but construction gets angry when an incomplete object is flushed.
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~wgrant/launchpad/gpgkey-fks into lp:launchpad.
=== modified file 'database/schema/security.cfg'
--- database/schema/security.cfg 2016-03-02 21:21:26 +0000
+++ database/schema/security.cfg 2016-03-03 18:43:25 +0000
@@ -2277,6 +2277,7 @@
public.accesspolicygrant = SELECT, DELETE
public.account = SELECT, DELETE
public.answercontact = SELECT, DELETE
+public.archive = SELECT, UPDATE
public.branch = SELECT, UPDATE
public.branchjob = SELECT, DELETE
public.binarypackagename = SELECT
@@ -2318,6 +2319,7 @@
public.milestonetag = SELECT
public.openidconsumerassociation = SELECT, DELETE
public.openidconsumernonce = SELECT, DELETE
+public.packageupload = SELECT, UPDATE
public.person = SELECT, DELETE
public.personsettings = SELECT, UPDATE
public.product = SELECT, UPDATE
@@ -2327,9 +2329,10 @@
public.previewdiff = SELECT, DELETE
public.revisionauthor = SELECT, UPDATE
public.revisioncache = SELECT, DELETE
+public.signedcodeofconduct = SELECT, UPDATE
public.snapfile = SELECT, DELETE
public.sourcepackagename = SELECT
-public.sourcepackagerelease = SELECT
+public.sourcepackagerelease = SELECT, UPDATE
public.sourcepackagepublishinghistory = SELECT, UPDATE
public.suggestivepotemplate = INSERT, DELETE
public.teammembership = SELECT, DELETE
=== modified file 'lib/lp/archivepublisher/archivesigningkey.py'
--- lib/lp/archivepublisher/archivesigningkey.py 2015-09-13 18:30:51 +0000
+++ lib/lp/archivepublisher/archivesigningkey.py 2016-03-03 18:43:25 +0000
@@ -70,7 +70,10 @@
if self.archive != default_ppa:
if default_ppa.signing_key is None:
IArchiveSigningKey(default_ppa).generateSigningKey()
- self.archive.signing_key = default_ppa.signing_key
+ key = default_ppa.signing_key
+ self.archive.signing_key = key
+ self.archive.signing_key_owner = key.owner
+ self.archive._signing_key_fingerprint = key.fingerprint
return
key_displayname = (
@@ -105,9 +108,12 @@
algorithm = GPGKeyAlgorithm.items[pub_key.algorithm]
key_owner = getUtility(ILaunchpadCelebrities).ppa_key_guard
- self.archive.signing_key = getUtility(IGPGKeySet).new(
+ key = getUtility(IGPGKeySet).new(
key_owner, pub_key.keyid, pub_key.fingerprint, pub_key.keysize,
algorithm, active=True, can_encrypt=pub_key.can_encrypt)
+ self.archive.signing_key = key
+ self.archive.signing_key_owner = key.owner
+ self.archive._signing_key_fingerprint = key.fingerprint
def signRepository(self, suite):
"""See `IArchiveSigningKey`."""
=== modified file 'lib/lp/registry/model/codeofconduct.py'
--- lib/lp/registry/model/codeofconduct.py 2015-07-10 15:31:28 +0000
+++ lib/lp/registry/model/codeofconduct.py 2016-03-03 18:43:25 +0000
@@ -19,6 +19,7 @@
ForeignKey,
StringCol,
)
+from storm.properties import Unicode
from zope.component import getUtility
from zope.interface import implementer
@@ -181,6 +182,7 @@
signingkey = ForeignKey(foreignKey="GPGKey", dbName="signingkey",
notNull=False, default=None)
+ signing_key_fingerprint = Unicode()
datecreated = UtcDateTimeCol(dbName='datecreated', notNull=True,
default=UTC_NOW)
@@ -305,8 +307,10 @@
'space differences are acceptable).')
# Store the signature
- signed = SignedCodeOfConduct(owner=user, signingkey=gpg,
- signedcode=signedcode, active=True)
+ signed = SignedCodeOfConduct(
+ owner=user, signingkey=gpg,
+ signing_key_fingerprint=gpg.fingerprint if gpg else None,
+ signedcode=signedcode, active=True)
# Send Advertisement Email
subject = 'Your Code of Conduct signature has been acknowledged'
=== modified file 'lib/lp/registry/model/distroseries.py'
--- lib/lp/registry/model/distroseries.py 2016-02-05 20:28:29 +0000
+++ lib/lp/registry/model/distroseries.py 2016-03-03 18:43:25 +0000
@@ -1164,8 +1164,11 @@
architecturehintlist=architecturehintlist, component=component,
creator=creator, urgency=urgency, changelog=changelog,
changelog_entry=changelog_entry, dsc=dsc,
- dscsigningkey=dscsigningkey, section=section, copyright=copyright,
- upload_archive=archive,
+ dscsigningkey=dscsigningkey,
+ signing_key_owner=dscsigningkey.owner if dscsigningkey else None,
+ signing_key_fingerprint=(
+ dscsigningkey.fingerprint if dscsigningkey else None),
+ section=section, copyright=copyright, upload_archive=archive,
dsc_maintainer_rfc822=dsc_maintainer_rfc822,
dsc_standards_version=dsc_standards_version,
dsc_format=dsc_format, dsc_binaries=dsc_binaries,
@@ -1350,7 +1353,11 @@
return PackageUpload(
distroseries=self, status=PackageUploadStatus.NEW,
pocket=pocket, archive=archive, changesfile=changes_file_alias,
- signing_key=signing_key, package_copy_job=package_copy_job)
+ signing_key=signing_key,
+ signing_key_owner=signing_key.owner if signing_key else None,
+ signing_key_fingerprint=(
+ signing_key.fingerprint if signing_key else None),
+ package_copy_job=package_copy_job)
def getPackageUploadQueue(self, state):
"""See `IDistroSeries`."""
=== modified file 'lib/lp/scripts/garbo.py'
--- lib/lp/scripts/garbo.py 2016-02-05 17:18:57 +0000
+++ lib/lp/scripts/garbo.py 2016-03-03 18:43:25 +0000
@@ -69,6 +69,7 @@
RevisionCache,
)
from lp.hardwaredb.model.hwdb import HWSubmission
+from lp.registry.model.codeofconduct import SignedCodeOfConduct
from lp.registry.model.commercialsubscription import CommercialSubscription
from lp.registry.model.person import (
Person,
@@ -124,6 +125,7 @@
from lp.soyuz.model.archive import Archive
from lp.soyuz.model.livefsbuild import LiveFSFile
from lp.soyuz.model.publishing import SourcePackagePublishingHistory
+from lp.soyuz.model.queue import PackageUpload
from lp.soyuz.model.reporting import LatestPersonSourcePackageReleaseCache
from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
from lp.translations.interfaces.potemplate import IPOTemplateSet
@@ -1461,6 +1463,78 @@
transaction.commit()
+class BaseKeyMigrator(TunableLoop):
+
+ maximum_chunk_size = 5000
+
+ def __init__(self, log, abort_time=None):
+ super(BaseKeyMigrator, self).__init__(
+ log, abort_time)
+ state = load_garbo_job_state(self.__class__.__name__) or {}
+ self.start_at = state.get('next_id', 1)
+ self.store = IMasterStore(self.klass)
+
+ def findObjects(self):
+ return self.store.find(
+ self.klass,
+ self.klass.id >= self.start_at).order_by(
+ self.klass.id)
+
+ def isDone(self):
+ return (
+ not getFeatureFlag('gpg.migrator.%s' % self.klass.__name__)
+ or self.findObjects().is_empty())
+
+ def __call__(self, chunk_size):
+ objs = list(self.findObjects()[:chunk_size])
+ for obj in objs:
+ key = getattr(obj, self.fk_attr, None)
+ if self.fingerprint_attr:
+ setattr(
+ obj, self.fingerprint_attr,
+ key.fingerprint if key else None)
+ if self.owner_attr:
+ setattr(
+ obj, self.owner_attr,
+ key.owner if key else None)
+ self.start_at = objs[-1].id + 1
+ save_garbo_job_state(
+ self.__class__.__name__, {'next_id': self.start_at})
+ transaction.commit()
+
+
+class ArchiveKeyMigrator(BaseKeyMigrator):
+
+ klass = Archive
+ fk_attr = 'signing_key'
+ fingerprint_attr = '_signing_key_fingerprint'
+ owner_attr = 'signing_key_owner'
+
+
+class PackageUploadKeyMigrator(BaseKeyMigrator):
+
+ klass = PackageUpload
+ fk_attr = 'signing_key'
+ fingerprint_attr = 'signing_key_fingerprint'
+ owner_attr = 'signing_key_owner'
+
+
+class SignedCodeOfConductKeyMigrator(BaseKeyMigrator):
+
+ klass = SignedCodeOfConduct
+ fk_attr = 'signingkey'
+ fingerprint_attr = 'signing_key_fingerprint'
+ owner_attr = None
+
+
+class SourcePackageReleaseKeyMigrator(BaseKeyMigrator):
+
+ klass = SourcePackageRelease
+ fk_attr = 'dscsigningkey'
+ fingerprint_attr = 'signing_key_fingerprint'
+ owner_attr = 'signing_key_owner'
+
+
class BaseDatabaseGarbageCollector(LaunchpadCronScript):
"""Abstract base class to run a collection of TunableLoops."""
script_name = None # Script name for locking and database user. Override.
@@ -1709,10 +1783,14 @@
"""
script_name = 'garbo-hourly'
tunable_loops = [
+ ArchiveKeyMigrator,
BugHeatUpdater,
BugWatchScheduler,
DuplicateSessionPruner,
+ PackageUploadKeyMigrator,
RevisionCachePruner,
+ SignedCodeOfConductKeyMigrator,
+ SourcePackageReleaseKeyMigrator,
UnusedSessionPruner,
]
experimental_tunable_loops = []
=== modified file 'lib/lp/scripts/tests/test_garbo.py'
--- lib/lp/scripts/tests/test_garbo.py 2016-02-06 02:55:51 +0000
+++ lib/lp/scripts/tests/test_garbo.py 2016-03-03 18:43:25 +0000
@@ -73,6 +73,7 @@
from lp.registry.interfaces.accesspolicy import IAccessPolicySource
from lp.registry.interfaces.person import IPersonSet
from lp.registry.interfaces.teammembership import TeamMembershipStatus
+from lp.registry.model.codeofconduct import SignedCodeOfConduct
from lp.registry.model.commercialsubscription import CommercialSubscription
from lp.registry.model.person import PersonSettings
from lp.registry.model.teammembership import TeamMembership
@@ -128,7 +129,10 @@
TestCase,
TestCaseWithFactory,
)
-from lp.testing.dbuser import switch_dbuser
+from lp.testing.dbuser import (
+ dbuser,
+ switch_dbuser,
+ )
from lp.testing.layers import (
DatabaseLayer,
LaunchpadScriptLayer,
@@ -1390,6 +1394,67 @@
self._test_LiveFSFilePruner(
'application/octet-stream', 0, expected_count=1)
+ def test_ArchiveKeyMigrator(self):
+ with dbuser('testadmin'):
+ key = self.factory.makeGPGKey(self.factory.makePerson())
+ a1 = self.factory.makeArchive()
+ a2 = self.factory.makeArchive()
+ removeSecurityProxy(a2).signing_key = key
+ self.assertIs(None, a1._signing_key_fingerprint)
+ self.assertIs(None, a1.signing_key_owner)
+ self.assertIs(None, a2._signing_key_fingerprint)
+ self.assertIs(None, a2.signing_key_owner)
+ with FeatureFixture({'gpg.migrator.Archive': 'on'}):
+ self.runHourly()
+ self.assertIs(None, a1._signing_key_fingerprint)
+ self.assertIs(None, a1.signing_key_owner)
+ self.assertEqual(key.fingerprint, a2._signing_key_fingerprint)
+ self.assertEqual(key.owner, a2.signing_key_owner)
+
+ def test_PackageUploadKeyMigrator(self):
+ with dbuser('testadmin'):
+ key = self.factory.makeGPGKey(self.factory.makePerson())
+ pu1 = self.factory.makePackageUpload()
+ pu2 = self.factory.makePackageUpload(signing_key=key)
+ npu2 = removeSecurityProxy(pu2)
+ npu2.signing_key_fingerprint = npu2.signing_key_owner = None
+ self.assertIs(None, pu1.signing_key_fingerprint)
+ self.assertIs(None, pu1.signing_key_owner)
+ self.assertIs(None, pu2.signing_key_fingerprint)
+ self.assertIs(None, pu2.signing_key_owner)
+ with FeatureFixture({'gpg.migrator.PackageUpload': 'on'}):
+ self.runHourly()
+ self.assertIs(None, pu1.signing_key_fingerprint)
+ self.assertIs(None, pu1.signing_key_owner)
+ self.assertEqual(key.fingerprint, pu2.signing_key_fingerprint)
+ self.assertEqual(key.owner, pu2.signing_key_owner)
+
+ def test_SignedCodeOfConductKeyMigrator(self):
+ coc = SignedCodeOfConduct.get(1)
+ self.assertIs(None, coc.signing_key_fingerprint)
+ with FeatureFixture({'gpg.migrator.SignedCodeOfConduct': 'on'}):
+ self.runHourly()
+ self.assertEqual(
+ 'ABCDEF0123456789ABCDDCBA0000111112345678',
+ coc.signing_key_fingerprint)
+
+ def test_SourcePackageReleaseKeyMigrator(self):
+ with dbuser('testadmin'):
+ key = self.factory.makeGPGKey(self.factory.makePerson())
+ spr1 = self.factory.makeSourcePackageRelease()
+ spr2 = self.factory.makeSourcePackageRelease()
+ removeSecurityProxy(spr2).dscsigningkey = key
+ self.assertIs(None, spr1.signing_key_fingerprint)
+ self.assertIs(None, spr1.signing_key_owner)
+ self.assertIs(None, spr2.signing_key_fingerprint)
+ self.assertIs(None, spr2.signing_key_owner)
+ with FeatureFixture({'gpg.migrator.SourcePackageRelease': 'on'}):
+ self.runHourly()
+ self.assertIs(None, spr1.signing_key_fingerprint)
+ self.assertIs(None, spr1.signing_key_owner)
+ self.assertEqual(key.fingerprint, spr2.signing_key_fingerprint)
+ self.assertEqual(key.owner, spr2.signing_key_owner)
+
def test_PersonSettingsENFPopulator(self):
switch_dbuser('testadmin')
store = IMasterStore(PersonSettings)
=== modified file 'lib/lp/soyuz/configure.zcml'
--- lib/lp/soyuz/configure.zcml 2016-01-08 14:35:52 +0000
+++ lib/lp/soyuz/configure.zcml 2016-03-03 18:43:25 +0000
@@ -75,6 +75,7 @@
class="lp.soyuz.model.sourcepackagerelease.SourcePackageRelease">
<allow
interface="lp.soyuz.interfaces.sourcepackagerelease.ISourcePackageRelease"/>
+ <allow attributes="signing_key_fingerprint signing_key_owner" />
<!-- changelog needs to be updated when unembargoing -->
<require
permission="launchpad.Edit"
@@ -143,6 +144,8 @@
changesfile
changes_file_url
signing_key
+ signing_key_fingerprint
+ signing_key_owner
archive
sources
sourceFileUrls
@@ -386,7 +389,9 @@
set_schema="lp.soyuz.interfaces.archive.IArchiveRestricted"/>
<require
permission="launchpad.InternalScriptsOnly"
- set_attributes="distribution signing_key"/>
+ attributes="signing_key_owner _signing_key_fingerprint"
+ set_attributes="distribution signing_key signing_key_owner
+ _signing_key_fingerprint"/>
</class>
<adapter
for="lp.soyuz.interfaces.archive.IArchive"
=== modified file 'lib/lp/soyuz/model/archive.py'
--- lib/lp/soyuz/model/archive.py 2015-11-26 15:46:38 +0000
+++ lib/lp/soyuz/model/archive.py 2016-03-03 18:43:25 +0000
@@ -23,21 +23,22 @@
IntCol,
StringCol,
)
+from storm.base import Storm
from storm.expr import (
And,
+ Count,
Desc,
+ Join,
Not,
Or,
Select,
Sum,
)
-from storm.locals import (
- Count,
+from storm.properties import (
Int,
- Join,
- Reference,
- Storm,
+ Unicode,
)
+from storm.references import Reference
from storm.store import Store
from zope.component import (
getAdapter,
@@ -339,6 +340,9 @@
signing_key = ForeignKey(
foreignKey='GPGKey', dbName='signing_key', notNull=False)
+ signing_key_owner_id = Int(name="signing_key_owner")
+ signing_key_owner = Reference(signing_key_owner_id, 'Person.id')
+ _signing_key_fingerprint = Unicode(name="signing_key_fingerprint")
relative_build_score = IntCol(
dbName='relative_build_score', notNull=True, default=0)
@@ -2534,6 +2538,9 @@
owner=owner, distribution=distribution, name=name,
displayname=displayname, description=description,
purpose=purpose, publish=publish, signing_key=signing_key,
+ signing_key_owner=signing_key.owner if signing_key else None,
+ _signing_key_fingerprint=(
+ signing_key.fingerprint if signing_key else None),
require_virtualized=require_virtualized)
# Upon creation archives are enabled by default.
=== modified file 'lib/lp/soyuz/model/queue.py'
--- lib/lp/soyuz/model/queue.py 2016-02-05 16:51:12 +0000
+++ lib/lp/soyuz/model/queue.py 2016-03-03 18:43:25 +0000
@@ -186,6 +186,9 @@
signing_key = ForeignKey(
foreignKey='GPGKey', dbName='signing_key', notNull=False)
+ signing_key_owner_id = Int(name="signing_key_owner")
+ signing_key_owner = Reference(signing_key_owner_id, 'Person.id')
+ signing_key_fingerprint = Unicode()
package_copy_job_id = Int(name='package_copy_job', allow_none=True)
package_copy_job = Reference(package_copy_job_id, 'PackageCopyJob.id')
=== modified file 'lib/lp/soyuz/model/sourcepackagerelease.py'
--- lib/lp/soyuz/model/sourcepackagerelease.py 2016-02-05 15:16:29 +0000
+++ lib/lp/soyuz/model/sourcepackagerelease.py 2016-03-03 18:43:25 +0000
@@ -30,6 +30,7 @@
Desc,
Int,
Reference,
+ Unicode,
)
from storm.store import Store
from zope.component import getUtility
@@ -89,6 +90,9 @@
dbName='maintainer', foreignKey='Person',
storm_validator=validate_public_person, notNull=True)
dscsigningkey = ForeignKey(foreignKey='GPGKey', dbName='dscsigningkey')
+ signing_key_owner_id = Int(name="signing_key_owner")
+ signing_key_owner = Reference(signing_key_owner_id, 'Person.id')
+ signing_key_fingerprint = Unicode()
urgency = EnumCol(dbName='urgency', schema=SourcePackageUrgency,
default=SourcePackageUrgency.LOW, notNull=True)
dateuploaded = UtcDateTimeCol(dbName='dateuploaded', notNull=True,
=== modified file 'lib/lp/soyuz/scripts/gina/handlers.py'
--- lib/lp/soyuz/scripts/gina/handlers.py 2015-05-20 11:31:11 +0000
+++ lib/lp/soyuz/scripts/gina/handlers.py 2016-03-03 18:43:25 +0000
@@ -616,6 +616,8 @@
sourcepackagename=name.id,
maintainer=maintainer.id,
dscsigningkey=key,
+ signing_key_owner=key.owner if key else None,
+ signing_key_fingerprint=key.fingerprint if key else None,
urgency=ChangesFile.urgency_map[src.urgency],
dateuploaded=src.date_uploaded,
dsc=src.dsc,
Follow ups
