launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #21048
[Merge] lp:~cjwatson/launchpad/git-repository-type into lp:launchpad
Colin Watson has proposed merging lp:~cjwatson/launchpad/git-repository-type into lp:launchpad with lp:~cjwatson/launchpad/codeimport-mail-unique-name as a prerequisite.
Commit message:
Model GitRepository.repository_type; add garbo job to backfill it with HOSTED.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/git-repository-type/+merge/307464
Model GitRepository.repository_type; add garbo job to backfill it with HOSTED.
Requires https://code.launchpad.net/~cjwatson/launchpad/db-git-imports/+merge/307439.
--
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/git-repository-type into lp:launchpad.
=== modified file 'database/schema/security.cfg'
--- database/schema/security.cfg 2016-10-02 00:33:51 +0000
+++ database/schema/security.cfg 2016-10-03 16:23:58 +0000
@@ -2370,6 +2370,7 @@
public.emailaddress = SELECT, UPDATE, DELETE
public.garbojobstate = SELECT, INSERT, UPDATE, DELETE
public.gitjob = SELECT, DELETE
+public.gitrepository = SELECT, UPDATE
public.hwsubmission = SELECT, UPDATE
public.job = SELECT, INSERT, DELETE
public.latestpersonsourcepackagereleasecache = SELECT, INSERT, UPDATE
=== modified file 'lib/lp/code/enums.py'
--- lib/lp/code/enums.py 2015-09-28 17:38:45 +0000
+++ lib/lp/code/enums.py 2016-10-03 16:23:58 +0000
@@ -21,6 +21,7 @@
'CodeReviewNotificationLevel',
'CodeReviewVote',
'GitObjectType',
+ 'GitRepositoryType',
'NON_CVS_RCS_TYPES',
'RevisionControlSystems',
]
@@ -78,7 +79,7 @@
class BranchType(DBEnumeratedType):
"""Branch Type
- The type of a branch determins the branch interaction with a number
+ The type of a branch determines the branch's interaction with a number
of other subsystems.
"""
@@ -110,6 +111,30 @@
""")
+class GitRepositoryType(DBEnumeratedType):
+ """Git Repository Type
+
+ The type of a repository determines its interaction with other
+ subsystems.
+ """
+
+ HOSTED = DBItem(1, """
+ Hosted
+
+ Launchpad is the primary location of this repository.
+ """)
+
+ # Skipping MIRRORED (2) to stay in sync with BranchType, in order to
+ # reduce confusion for manual database queries.
+
+ IMPORTED = DBItem(3, """
+ Imported
+
+ Repositories that have been imported from an externally hosted
+ repository and are made available through Launchpad.
+ """)
+
+
class GitObjectType(DBEnumeratedType):
"""Git Object Type
=== modified file 'lib/lp/code/interfaces/gitnamespace.py'
--- lib/lp/code/interfaces/gitnamespace.py 2016-01-12 13:43:34 +0000
+++ lib/lp/code/interfaces/gitnamespace.py 2016-10-03 16:23:58 +0000
@@ -33,8 +33,8 @@
target = Attribute("The `IHasGitRepositories` for this namespace.")
- def createRepository(registrant, name, information_type=None,
- date_created=None):
+ def createRepository(repository_type, registrant, name,
+ information_type=None, date_created=None):
"""Create and return an `IGitRepository` in this namespace."""
def isNameUsed(name):
=== modified file 'lib/lp/code/interfaces/gitrepository.py'
--- lib/lp/code/interfaces/gitrepository.py 2016-09-06 15:34:38 +0000
+++ lib/lp/code/interfaces/gitrepository.py 2016-10-03 16:23:58 +0000
@@ -61,6 +61,7 @@
BranchSubscriptionDiffSize,
BranchSubscriptionNotificationLevel,
CodeReviewNotificationLevel,
+ GitRepositoryType,
)
from lp.code.interfaces.defaultgit import ICanHasDefaultGitRepository
from lp.code.interfaces.hasgitrepositories import IHasGitRepositories
@@ -127,6 +128,10 @@
date_created = exported(Datetime(
title=_("Date created"), required=True, readonly=True))
+ repository_type = exported(Choice(
+ title=_("Repository type"), required=True, readonly=True,
+ vocabulary=GitRepositoryType))
+
registrant = exported(PublicPersonChoice(
title=_("Registrant"), required=True, readonly=True,
vocabulary="ValidPersonOrTeam",
=== modified file 'lib/lp/code/model/gitnamespace.py'
--- lib/lp/code/model/gitnamespace.py 2016-01-12 13:43:34 +0000
+++ lib/lp/code/model/gitnamespace.py 2016-10-03 16:23:58 +0000
@@ -70,9 +70,9 @@
class _BaseGitNamespace:
"""Common code for Git repository namespaces."""
- def createRepository(self, registrant, name, reviewer=None,
- information_type=None, date_created=DEFAULT,
- description=None):
+ def createRepository(self, repository_type, registrant, name,
+ reviewer=None, information_type=None,
+ date_created=DEFAULT, description=None):
"""See `IGitNamespace`."""
self.validateRegistrant(registrant)
@@ -84,8 +84,9 @@
raise GitRepositoryCreationForbidden()
repository = GitRepository(
- registrant, self.owner, self.target, name, information_type,
- date_created, reviewer=reviewer, description=description)
+ repository_type, registrant, self.owner, self.target, name,
+ information_type, date_created, reviewer=reviewer,
+ description=description)
repository._reconcileAccess()
# The owner of the repository should also be automatically subscribed
=== modified file 'lib/lp/code/model/gitrepository.py'
--- lib/lp/code/model/gitrepository.py 2016-09-06 15:34:38 +0000
+++ lib/lp/code/model/gitrepository.py 2016-10-03 16:23:58 +0000
@@ -70,7 +70,10 @@
IPrivacy,
)
from lp.app.interfaces.services import IService
-from lp.code.enums import GitObjectType
+from lp.code.enums import (
+ GitObjectType,
+ GitRepositoryType,
+ )
from lp.code.errors import (
CannotDeleteGitRepository,
GitDefaultConflict,
@@ -238,6 +241,22 @@
date_last_modified = DateTime(
name='date_last_modified', tzinfo=pytz.UTC, allow_none=False)
+ _repository_type = EnumCol(
+ dbName='repository_type', enum=GitRepositoryType, notNull=False)
+
+ @property
+ def repository_type(self):
+ # XXX cjwatson 2016-10-03: Remove once this column has been
+ # backfilled.
+ if self._repository_type is None:
+ return GitRepositoryType.HOSTED
+ else:
+ return self._repository_type
+
+ @repository_type.setter
+ def repository_type(self, value):
+ self._repository_type = value
+
registrant_id = Int(name='registrant', allow_none=False)
registrant = Reference(registrant_id, 'Person.id')
@@ -266,9 +285,11 @@
_default_branch = Unicode(name='default_branch', allow_none=True)
- def __init__(self, registrant, owner, target, name, information_type,
- date_created, reviewer=None, description=None):
+ def __init__(self, repository_type, registrant, owner, target, name,
+ information_type, date_created, reviewer=None,
+ description=None):
super(GitRepository, self).__init__()
+ self.repository_type = repository_type
self.registrant = registrant
self.owner = owner
self.reviewer = reviewer
=== modified file 'lib/lp/code/model/tests/test_gitnamespace.py'
--- lib/lp/code/model/tests/test_gitnamespace.py 2015-10-09 15:49:14 +0000
+++ lib/lp/code/model/tests/test_gitnamespace.py 2016-10-03 16:23:58 +0000
@@ -15,6 +15,7 @@
from lp.app.interfaces.launchpad import ILaunchpadCelebrities
from lp.app.interfaces.services import IService
from lp.app.validators import LaunchpadValidationError
+from lp.code.enums import GitRepositoryType
from lp.code.errors import (
GitDefaultConflict,
GitRepositoryCreatorNotMemberOfOwnerTeam,
@@ -63,7 +64,8 @@
namespace = self.getNamespace()
repository_name = self.factory.getUniqueUnicode()
registrant = removeSecurityProxy(namespace).owner
- repository = namespace.createRepository(registrant, repository_name)
+ repository = namespace.createRepository(
+ GitRepositoryType.HOSTED, registrant, repository_name)
self.assertEqual(
"%s/+git/%s" % (namespace.name, repository_name),
repository.unique_name)
@@ -78,8 +80,9 @@
reviewer = self.factory.makePerson()
description = self.factory.getUniqueUnicode()
repository = namespace.createRepository(
- registrant, repository_name, reviewer=reviewer,
- description=description)
+ GitRepositoryType.HOSTED, registrant, repository_name,
+ reviewer=reviewer, description=description)
+ self.assertEqual(GitRepositoryType.HOSTED, repository.repository_type)
self.assertEqual(repository_name, repository.name)
self.assertEqual(registrant, repository.registrant)
self.assertEqual(reviewer, repository.reviewer)
@@ -90,7 +93,8 @@
namespace = self.getNamespace(owner)
repository_name = self.factory.getUniqueUnicode()
registrant = owner.teamowner
- repository = namespace.createRepository(registrant, repository_name)
+ repository = namespace.createRepository(
+ GitRepositoryType.HOSTED, registrant, repository_name)
self.assertEqual([owner], list(repository.subscribers))
def test_getRepositories_no_repositories(self):
@@ -106,7 +110,8 @@
namespace = self.getNamespace()
repository_name = self.factory.getUniqueUnicode()
repository = namespace.createRepository(
- removeSecurityProxy(namespace).owner, repository_name)
+ GitRepositoryType.HOSTED, removeSecurityProxy(namespace).owner,
+ repository_name)
self.assertEqual([repository], list(namespace.getRepositories()))
def test_getByName_default(self):
@@ -127,7 +132,8 @@
namespace = self.getNamespace()
repository_name = self.factory.getUniqueUnicode()
repository = namespace.createRepository(
- removeSecurityProxy(namespace).owner, repository_name)
+ GitRepositoryType.HOSTED, removeSecurityProxy(namespace).owner,
+ repository_name)
match = namespace.getByName(repository_name)
self.assertEqual(repository, match)
@@ -140,7 +146,8 @@
namespace = self.getNamespace()
repository_name = self.factory.getUniqueUnicode()
namespace.createRepository(
- removeSecurityProxy(namespace).owner, repository_name)
+ GitRepositoryType.HOSTED, removeSecurityProxy(namespace).owner,
+ repository_name)
self.assertTrue(namespace.isNameUsed(repository_name))
def test_findUnusedName_unused(self):
@@ -155,7 +162,9 @@
# it's already used.
namespace = self.getNamespace()
name = self.factory.getUniqueUnicode()
- namespace.createRepository(removeSecurityProxy(namespace).owner, name)
+ namespace.createRepository(
+ GitRepositoryType.HOSTED, removeSecurityProxy(namespace).owner,
+ name)
unused_name = namespace.findUnusedName(name)
self.assertEqual("%s-1" % name, unused_name)
@@ -164,9 +173,12 @@
# it's already used.
namespace = self.getNamespace()
name = self.factory.getUniqueUnicode()
- namespace.createRepository(removeSecurityProxy(namespace).owner, name)
- namespace.createRepository(
- removeSecurityProxy(namespace).owner, name + "-1")
+ namespace.createRepository(
+ GitRepositoryType.HOSTED, removeSecurityProxy(namespace).owner,
+ name)
+ namespace.createRepository(
+ GitRepositoryType.HOSTED, removeSecurityProxy(namespace).owner,
+ name + "-1")
unused_name = namespace.findUnusedName(name)
self.assertEqual("%s-2" % name, unused_name)
@@ -187,7 +199,9 @@
namespace = self.getNamespace()
namespace_owner = removeSecurityProxy(namespace).owner
name = self.factory.getUniqueUnicode()
- namespace.createRepository(removeSecurityProxy(namespace).owner, name)
+ namespace.createRepository(
+ GitRepositoryType.HOSTED, removeSecurityProxy(namespace).owner,
+ name)
repository = self.factory.makeGitRepository(name=name)
self.assertRaises(
GitRepositoryExists,
@@ -224,7 +238,9 @@
namespace = self.getNamespace()
namespace_owner = removeSecurityProxy(namespace).owner
name = self.factory.getUniqueUnicode()
- namespace.createRepository(removeSecurityProxy(namespace).owner, name)
+ namespace.createRepository(
+ GitRepositoryType.HOSTED, removeSecurityProxy(namespace).owner,
+ name)
repository = self.factory.makeGitRepository()
self.assertRaises(
GitRepositoryExists,
@@ -454,7 +470,7 @@
namespace.target, team, namespace.target.owner,
{InformationType.PROPRIETARY: SharingPermission.ALL})
repository = namespace.createRepository(
- person, self.factory.getUniqueUnicode())
+ GitRepositoryType.HOSTED, person, self.factory.getUniqueUnicode())
[policy] = getUtility(IAccessPolicySource).find(
[(namespace.target, InformationType.PROPRIETARY)])
apgfs = getUtility(IAccessPolicyGrantFlatSource)
@@ -598,7 +614,8 @@
# GitRepositoryExists is raised.
namespace = self._getNamespace(self.factory.makePerson())
repository = namespace.createRepository(
- namespace.owner, self.factory.getUniqueUnicode())
+ GitRepositoryType.HOSTED, namespace.owner,
+ self.factory.getUniqueUnicode())
self.assertRaises(
GitRepositoryExists,
namespace.validateRepositoryName, repository.name)
=== modified file 'lib/lp/code/xmlrpc/git.py'
--- lib/lp/code/xmlrpc/git.py 2015-07-10 04:51:21 +0000
+++ lib/lp/code/xmlrpc/git.py 2016-10-03 16:23:58 +0000
@@ -20,6 +20,7 @@
from lp.app.errors import NameLookupFailed
from lp.app.validators import LaunchpadValidationError
+from lp.code.enums import GitRepositoryType
from lp.code.errors import (
GitRepositoryCreationException,
GitRepositoryCreationFault,
@@ -171,7 +172,7 @@
try:
repository = namespace.createRepository(
- requester, repository_name)
+ GitRepositoryType.HOSTED, requester, repository_name)
except LaunchpadValidationError as e:
# Despite the fault name, this just passes through the exception
# text so there's no need for a new Git-specific fault.
=== modified file 'lib/lp/scripts/garbo.py'
--- lib/lp/scripts/garbo.py 2016-07-26 22:43:23 +0000
+++ lib/lp/scripts/garbo.py 2016-10-03 16:23:58 +0000
@@ -41,6 +41,7 @@
Or,
Row,
SQL,
+ Update,
)
from storm.info import ClassAlias
from storm.store import EmptyResultSet
@@ -58,6 +59,7 @@
BugWatchScheduler,
MAX_SAMPLE_SIZE,
)
+from lp.code.enums import GitRepositoryType
from lp.code.interfaces.revision import IRevisionSet
from lp.code.model.codeimportevent import CodeImportEvent
from lp.code.model.codeimportresult import CodeImportResult
@@ -65,6 +67,7 @@
Diff,
PreviewDiff,
)
+from lp.code.model.gitrepository import GitRepository
from lp.code.model.revision import (
RevisionAuthor,
RevisionCache,
@@ -1603,6 +1606,33 @@
transaction.commit()
+class GitRepositoryTypePopulator(TunableLoop):
+ """Populates GitRepository.repository_type with HOSTED."""
+
+ maximum_chunk_size = 5000
+
+ def __init__(self, log, abort_time=None):
+ super(GitRepositoryTypePopulator, self).__init__(log, abort_time)
+ self.start_at = 1
+ self.store = IMasterStore(GitRepository)
+
+ def findRepositories(self):
+ return self.store.find(
+ GitRepository,
+ GitRepository.id >= self.start_at,
+ GitRepository._repository_type == None).order_by(GitRepository.id)
+
+ def isDone(self):
+ return self.findRepositories().is_empty()
+
+ def __call__(self, chunk_size):
+ ids = [repository.id for repository in self.findRepositories()]
+ self.store.execute(Update(
+ {GitRepository._repository_type: GitRepositoryType.HOSTED.value},
+ where=GitRepository.id.is_in(ids), table=GitRepository))
+ transaction.commit()
+
+
class BaseDatabaseGarbageCollector(LaunchpadCronScript):
"""Abstract base class to run a collection of TunableLoops."""
script_name = None # Script name for locking and database user. Override.
@@ -1810,8 +1840,9 @@
tunable_loop.run()
loop_logger.debug(
"%s completed sucessfully.", loop_name)
- except Exception:
+ except Exception as e:
loop_logger.exception("Unhandled exception")
+ print(e)
self.failure_count += 1
finally:
@@ -1883,6 +1914,7 @@
CodeImportResultPruner,
DiffPruner,
GitJobPruner,
+ GitRepositoryTypePopulator,
HWSubmissionEmailLinker,
LiveFSFilePruner,
LoginTokenPruner,
=== modified file 'lib/lp/scripts/tests/test_garbo.py'
--- lib/lp/scripts/tests/test_garbo.py 2016-07-26 22:43:23 +0000
+++ lib/lp/scripts/tests/test_garbo.py 2016-10-03 16:23:58 +0000
@@ -15,6 +15,7 @@
from StringIO import StringIO
import time
+from psycopg2 import IntegrityError
from pytz import UTC
from storm.exceptions import LostObjectError
from storm.expr import (
@@ -48,7 +49,10 @@
BranchFormat,
RepositoryFormat,
)
-from lp.code.enums import CodeImportResultStatus
+from lp.code.enums import (
+ CodeImportResultStatus,
+ GitRepositoryType,
+ )
from lp.code.interfaces.codeimportevent import ICodeImportEventSet
from lp.code.interfaces.gitrepository import IGitRepositorySet
from lp.code.model.branchjob import (
@@ -1553,6 +1557,44 @@
# Snaps with more than one possible store series are untouched.
self.assertIsNone(snaps[5].store_series)
+ def test_GitRepositoryTypePopulator(self):
+ switch_dbuser('testadmin')
+ old_repositories = [self.factory.makeGitRepository() for _ in range(2)]
+ for repository in old_repositories:
+ removeSecurityProxy(repository)._repository_type = None
+ try:
+ Store.of(old_repositories[0]).flush()
+ except IntegrityError:
+ # Now enforced by DB NOT NULL constraint; backfilling is no
+ # longer necessary.
+ return
+ hosted_repositories = [
+ self.factory.makeGitRepository(
+ repository_type=GitRepositoryType.HOSTED)
+ for _ in range(2)]
+ imported_repositories = [
+ self.factory.makeGitRepository(
+ repository_type=GitRepositoryType.IMPORTED)
+ for _ in range(2)]
+ transaction.commit()
+
+ self.runDaily()
+
+ # Old repositories are backfilled.
+ for repository in old_repositories:
+ self.assertEqual(
+ GitRepositoryType.HOSTED,
+ removeSecurityProxy(repository)._repository_type)
+ # Other repositories are left alone.
+ for repository in hosted_repositories:
+ self.assertEqual(
+ GitRepositoryType.HOSTED,
+ removeSecurityProxy(repository)._repository_type)
+ for repository in imported_repositories:
+ self.assertEqual(
+ GitRepositoryType.IMPORTED,
+ removeSecurityProxy(repository)._repository_type)
+
class TestGarboTasks(TestCaseWithFactory):
layer = LaunchpadZopelessLayer
=== modified file 'lib/lp/testing/factory.py'
--- lib/lp/testing/factory.py 2016-09-30 13:27:27 +0000
+++ lib/lp/testing/factory.py 2016-10-03 16:23:58 +0000
@@ -112,6 +112,7 @@
CodeImportReviewStatus,
CodeReviewNotificationLevel,
GitObjectType,
+ GitRepositoryType,
RevisionControlSystems,
)
from lp.code.errors import UnknownBranchTypeError
@@ -1754,14 +1755,17 @@
revision_date=revision_date)
return branch.createBranchRevision(sequence, revision)
- def makeGitRepository(self, owner=None, reviewer=None, target=_DEFAULT,
- registrant=None, name=None, information_type=None,
+ def makeGitRepository(self, repository_type=None, owner=None,
+ reviewer=None, target=_DEFAULT, registrant=None,
+ name=None, information_type=None,
**optional_repository_args):
"""Create and return a new, arbitrary GitRepository.
Any parameters for `IGitNamespace.createRepository` can be specified
to override the default ones.
"""
+ if repository_type is None:
+ repository_type = GitRepositoryType.HOSTED
if owner is None:
owner = self.makePerson()
if name is None:
@@ -1778,8 +1782,8 @@
namespace = get_git_namespace(target, owner)
repository = namespace.createRepository(
- registrant=registrant, name=name, reviewer=reviewer,
- **optional_repository_args)
+ repository_type=repository_type, registrant=registrant, name=name,
+ reviewer=reviewer, **optional_repository_args)
naked_repository = removeSecurityProxy(repository)
if information_type is not None:
naked_repository.transitionToInformationType(
Follow ups
