launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #18152
Re: [Merge] lp:~cjwatson/launchpad/git-ref-scanner into lp:launchpad
Diff comments:
> === modified file 'lib/lp/code/configure.zcml'
> --- lib/lp/code/configure.zcml 2015-03-06 16:31:30 +0000
> +++ lib/lp/code/configure.zcml 2015-03-16 16:23:14 +0000
> @@ -859,6 +859,14 @@
> <allow interface="lp.code.interfaces.gitnamespace.IGitNamespaceSet" />
> </securedutility>
>
> + <!-- GitRef -->
> +
> + <class class="lp.code.model.gitref.GitRef">
> + <require
> + permission="launchpad.View"
> + interface="lp.code.interfaces.gitref.IGitRef" />
> + </class>
> +
> <!-- GitCollection -->
>
> <class class="lp.code.model.gitcollection.GenericGitCollection">
> @@ -933,6 +941,20 @@
> <adapter factory="lp.code.model.gitlookup.DistributionGitTraversable" />
> <adapter factory="lp.code.model.gitlookup.DistributionSourcePackageGitTraversable" />
>
> + <!-- Git-related jobs -->
> + <class class="lp.code.model.gitjob.GitJob">
> + <allow interface="lp.code.interfaces.gitjob.IGitJob" />
> + </class>
> + <securedutility
> + component="lp.code.model.gitjob.GitRefScanJob"
> + provides="lp.code.interfaces.gitjob.IGitRefScanJobSource">
> + <allow interface="lp.code.interfaces.gitjob.IGitRefScanJobSource" />
> + </securedutility>
> + <class class="lp.code.model.gitjob.GitRefScanJob">
> + <allow interface="lp.code.interfaces.gitjob.IGitJob" />
> + <allow interface="lp.code.interfaces.gitjob.IGitRefScanJob" />
> + </class>
> +
> <lp:help-folder folder="help" name="+help-code" />
>
> <!-- Diffs -->
>
> === modified file 'lib/lp/code/enums.py'
> --- lib/lp/code/enums.py 2014-02-24 07:19:52 +0000
> +++ lib/lp/code/enums.py 2015-03-16 16:23:14 +0000
> @@ -1,4 +1,4 @@
> -# Copyright 2009-2011 Canonical Ltd. This software is licensed under the
> +# Copyright 2009-2015 Canonical Ltd. This software is licensed under the
> # GNU Affero General Public License version 3 (see the file LICENSE).
>
> """Enumerations used in the lp/code modules."""
> @@ -20,6 +20,7 @@
> 'CodeImportReviewStatus',
> 'CodeReviewNotificationLevel',
> 'CodeReviewVote',
> + 'GitObjectType',
> 'NON_CVS_RCS_TYPES',
> 'RevisionControlSystems',
> 'UICreatableBranchType',
> @@ -115,6 +116,37 @@
> use_template(BranchType, exclude='IMPORTED')
>
>
> +class GitObjectType(DBEnumeratedType):
> + """Git Object Type
> +
> + Keep these in sync with the concrete GIT_OBJ_* enum values in libgit2.
> + """
> +
> + COMMIT = DBItem(1, """
> + Commit
> +
> + A commit object.
> + """)
> +
> + TREE = DBItem(2, """
> + Tree
> +
> + A tree (directory listing) object.
> + """)
> +
> + BLOB = DBItem(3, """
> + Blob
> +
> + A file revision object.
> + """)
> +
> + TAG = DBItem(4, """
> + Tag
> +
> + An annotated tag object.
> + """)
> +
> +
> class BranchLifecycleStatusFilter(EnumeratedType):
> """Branch Lifecycle Status Filter
>
>
> === modified file 'lib/lp/code/errors.py'
> --- lib/lp/code/errors.py 2015-03-04 18:27:40 +0000
> +++ lib/lp/code/errors.py 2015-03-16 16:23:14 +0000
> @@ -36,6 +36,7 @@
> 'GitRepositoryCreatorNotMemberOfOwnerTeam',
> 'GitRepositoryCreatorNotOwner',
> 'GitRepositoryExists',
> + 'GitRepositoryRefScanFault',
> 'GitTargetError',
> 'InvalidBranchMergeProposal',
> 'InvalidMergeQueueConfig',
> @@ -381,6 +382,10 @@
> """Raised when there is a hosting fault creating a Git repository."""
>
>
> +class GitRepositoryRefScanFault(Exception):
> + """Raised when there is a fault getting the refs for a repository."""
> +
> +
> class GitTargetError(Exception):
> """Raised when there is an error determining a Git repository target."""
>
>
> === modified file 'lib/lp/code/githosting.py'
> --- lib/lp/code/githosting.py 2015-03-03 14:51:10 +0000
> +++ lib/lp/code/githosting.py 2015-03-16 16:23:14 +0000
> @@ -9,11 +9,14 @@
> ]
>
> import json
> -from urlparse import urljoin
>
> +from bzrlib import urlutils
> import requests
>
> -from lp.code.errors import GitRepositoryCreationFault
> +from lp.code.errors import (
> + GitRepositoryCreationFault,
> + GitRepositoryRefScanFault,
> + )
>
>
> class GitHostingClient:
> @@ -40,7 +43,7 @@
> # should just use post(json=) and drop the explicit Content-Type
> # header.
> response = self._makeSession().post(
> - urljoin(self.endpoint, "repo"),
> + urlutils.join(self.endpoint, "repo"),
> headers={"Content-Type": "application/json"},
> data=json.dumps({"repo_path": path, "bare_repo": True}),
> timeout=self.timeout)
> @@ -50,3 +53,20 @@
> if response.status_code != 200:
> raise GitRepositoryCreationFault(
> "Failed to create Git repository: %s" % response.text)
> +
> + def get_refs(self, path):
> + try:
> + response = self._makeSession().get(
> + urlutils.join(self.endpoint, "repo", path, "refs"),
> + timeout=self.timeout)
> + except Exception as e:
> + raise GitRepositoryRefScanFault(
> + "Failed to get refs from Git repository: %s" % unicode(e))
> + if response.status_code != 200:
> + raise GitRepositoryRefScanFault(
> + "Failed to get refs from Git repository: %s" % response.text)
> + try:
> + return response.json()
> + except ValueError as e:
> + raise GitRepositoryRefScanFault(
> + "Failed to decode ref-scan response: %s" % unicode(e))
>
> === modified file 'lib/lp/code/interfaces/gitapi.py'
> --- lib/lp/code/interfaces/gitapi.py 2015-03-04 11:12:06 +0000
> +++ lib/lp/code/interfaces/gitapi.py 2015-03-16 16:23:14 +0000
> @@ -49,3 +49,14 @@
> "writable", whose value is True if the requester can push to
> this repository, otherwise False.
> """
> +
> + def notify(translated_path):
> + """Notify of a change to the repository at 'translated_path'.
> +
> + :param translated_path: The translated path to the repository. (We
> + use translated paths here in order to avoid problems with
> + repository names etc. being changed during a push.)
> +
> + :returns: A `NotFound` fault if no repository can be found for
> + 'translated_path'; otherwise None.
> + """
>
> === added file 'lib/lp/code/interfaces/gitjob.py'
> --- lib/lp/code/interfaces/gitjob.py 1970-01-01 00:00:00 +0000
> +++ lib/lp/code/interfaces/gitjob.py 2015-03-16 16:23:14 +0000
> @@ -0,0 +1,53 @@
> +# Copyright 2015 Canonical Ltd. This software is licensed under the
> +# GNU Affero General Public License version 3 (see the file LICENSE).
> +
> +"""GitJob interfaces."""
> +
> +__metaclass__ = type
> +
> +__all__ = [
> + 'IGitJob',
> + 'IGitRefScanJob',
> + 'IGitRefScanJobSource',
> + ]
> +
> +from lazr.restful.fields import Reference
> +from zope.interface import (
> + Attribute,
> + Interface,
> + )
> +
> +from lp import _
> +from lp.code.interfaces.gitrepository import IGitRepository
> +from lp.services.job.interfaces.job import (
> + IJob,
> + IJobSource,
> + IRunnableJob,
> + )
> +
> +
> +class IGitJob(Interface):
> + """A job related to a Git repository."""
> +
> + job = Reference(
> + title=_("The common Job attributes."), schema=IJob,
> + required=True, readonly=True)
> +
> + repository = Reference(
> + title=_("The Git repository to use for this job."),
> + schema=IGitRepository, required=True, readonly=True)
> +
> + metadata = Attribute(_("A dict of data about the job."))
> +
> +
> +class IGitRefScanJob(IRunnableJob):
> + """A Job that scans a Git repository for its current list of references."""
> +
> +
> +class IGitRefScanJobSource(IJobSource):
> +
> + def create(repository):
> + """Scan a repository for refs.
> +
> + :param repository: The database repository to scan.
> + """
>
> === modified file 'lib/lp/code/interfaces/gitlookup.py'
> --- lib/lp/code/interfaces/gitlookup.py 2015-03-05 11:39:06 +0000
> +++ lib/lp/code/interfaces/gitlookup.py 2015-03-16 16:23:14 +0000
> @@ -90,6 +90,12 @@
> Return the default value if there is no such repository.
> """
>
> + def getByHostingPath(path):
> + """Get information about a given path on the hosting backend.
> +
> + :return: An `IGitRepository`, or None.
> + """
> +
> def getByUniqueName(unique_name):
> """Find a repository by its unique name.
>
>
> === added file 'lib/lp/code/interfaces/gitref.py'
> --- lib/lp/code/interfaces/gitref.py 1970-01-01 00:00:00 +0000
> +++ lib/lp/code/interfaces/gitref.py 2015-03-16 16:23:14 +0000
> @@ -0,0 +1,43 @@
> +# Copyright 2015 Canonical Ltd. This software is licensed under the
> +# GNU Affero General Public License version 3 (see the file LICENSE).
> +
> +"""Git reference ("ref") interfaces."""
> +
> +__metaclass__ = type
> +
> +__all__ = [
> + 'IGitRef',
> + ]
> +
> +from zope.interface import (
> + Attribute,
> + Interface,
> + )
> +from zope.schema import (
> + Choice,
> + TextLine,
> + )
> +
> +from lp import _
> +from lp.code.enums import GitObjectType
> +
> +
> +class IGitRef(Interface):
> + """A reference in a Git repository."""
> +
> + repository = Attribute("The Git repository containing this reference.")
> +
> + path = TextLine(
> + title=_("Path"), required=True, readonly=True,
> + description=_(
> + "The full path of this reference, e.g. refs/heads/master."))
> +
> + commit_sha1 = TextLine(
> + title=_("Commit SHA-1"), required=True, readonly=True,
> + description=_(
> + "The full SHA-1 object name of the commit object referenced by "
> + "this reference."))
> +
> + object_type = Choice(
> + title=_("Object type"), required=True, readonly=True,
> + vocabulary=GitObjectType)
>
> === modified file 'lib/lp/code/interfaces/gitrepository.py'
> --- lib/lp/code/interfaces/gitrepository.py 2015-03-06 16:31:30 +0000
> +++ lib/lp/code/interfaces/gitrepository.py 2015-03-16 16:23:14 +0000
> @@ -186,6 +186,40 @@
> "'lp:' plus a shortcut version of the path via that target. "
> "Otherwise it is simply 'lp:' plus the unique name.")))
>
> + refs = Attribute("The references present in this repository.")
> +
> + def getRefByPath(path):
> + """Look up a single reference in this repository by path.
> +
> + :param path: A string to look up as a path.
> +
> + :return: An `IGitRef`, or None.
> + """
> +
> + def createOrUpdateRefs(refs_info, get_objects=False):
> + """Create or update a set of references in this repository.
> +
> + :param refs_info: A dict mapping ref paths to
> + {"sha1": sha1, "type": `GitObjectType`}.
> + :param get_objects: Return the created/updated references.
> +
> + :return: A list of the created/updated references if get_objects,
> + otherwise None.
> + """
> +
> + def removeRefs(paths):
> + """Remove a set of references in this repository.
> +
> + :params paths: An iterable of paths.
> + """
> +
> + def synchroniseRefs(hosting_refs):
> + """Synchronise references with those from the hosting service.
> +
> + :param hosting_refs: A dictionary of reference information returned
> + from the hosting service's `/repo/PATH/refs` collection.
> + """
> +
> def setOwnerDefault(value):
> """Set whether this repository is the default for its owner-target.
>
>
> === added file 'lib/lp/code/model/gitjob.py'
> --- lib/lp/code/model/gitjob.py 1970-01-01 00:00:00 +0000
> +++ lib/lp/code/model/gitjob.py 2015-03-16 16:23:14 +0000
> @@ -0,0 +1,197 @@
> +# Copyright 2015 Canonical Ltd. This software is licensed under the
> +# GNU Affero General Public License version 3 (see the file LICENSE).
> +
> +__metaclass__ = type
> +
> +__all__ = [
> + 'GitJob',
> + 'GitRefScanJob',
> + ]
> +
> +from lazr.delegates import delegates
> +from lazr.enum import (
> + DBEnumeratedType,
> + DBItem,
> + )
> +from storm.exceptions import LostObjectError
> +from storm.locals import (
> + Int,
> + JSON,
> + Reference,
> + Store,
> + )
> +from zope.interface import (
> + classProvides,
> + implements,
> + )
> +
> +from lp.app.errors import NotFoundError
> +from lp.code.githosting import GitHostingClient
> +from lp.code.interfaces.gitjob import (
> + IGitJob,
> + IGitRefScanJob,
> + IGitRefScanJobSource,
> + )
> +from lp.services.config import config
> +from lp.services.database.enumcol import EnumCol
> +from lp.services.database.interfaces import (
> + IMasterStore,
> + IStore,
> + )
> +from lp.services.database.locking import (
> + AdvisoryLockHeld,
> + LockType,
> + try_advisory_lock,
> + )
> +from lp.services.database.stormbase import StormBase
> +from lp.services.job.model.job import (
> + EnumeratedSubclass,
> + Job,
> + )
> +from lp.services.job.runner import BaseRunnableJob
> +from lp.services.mail.sendmail import format_address_for_person
> +from lp.services.scripts import log
> +
> +
> +class GitJobType(DBEnumeratedType):
> + """Values that `IGitJob.job_type` can take."""
> +
> + REF_SCAN = DBItem(0, """
> + Ref scan
> +
> + This job scans a repository for its current list of references.
> + """)
> +
> +
> +class GitJob(StormBase):
> + """See `IGitJob`."""
> +
> + __storm_table__ = 'GitJob'
> +
> + implements(IGitJob)
> +
> + job_id = Int(name='job', primary=True, allow_none=False)
> + job = Reference(job_id, 'Job.id')
> +
> + repository_id = Int(name='repository', allow_none=False)
> + repository = Reference(repository_id, 'GitRepository.id')
> +
> + job_type = EnumCol(enum=GitJobType, notNull=True)
> +
> + metadata = JSON('json_data')
> +
> + def __init__(self, repository, job_type, metadata, **job_args):
> + """Constructor.
> +
> + Extra keyword arguments are used to construct the underlying Job
> + object.
> +
> + :param repository: The database repository this job relates to.
> + :param job_type: The `GitJobType` of this job.
> + :param metadata: The type-specific variables, as a JSON-compatible
> + dict.
> + """
> + super(GitJob, self).__init__()
> + self.job = Job(**job_args)
> + self.repository = repository
> + self.job_type = job_type
> + self.metadata = metadata
> +
> + def makeDerived(self):
> + return GitJobDerived.makeSubclass(self)
> +
> +
> +class GitJobDerived(BaseRunnableJob):
> +
> + __metaclass__ = EnumeratedSubclass
> +
> + delegates(IGitJob)
> +
> + def __init__(self, git_job):
> + self.context = git_job
> +
> + @classmethod
> + def get(cls, job_id):
> + """Get a job by id.
> +
> + :return: The `GitJob` with the specified id, as the current
> + `GitJobDerived` subclass.
> + :raises: `NotFoundError` if there is no job with the specified id,
> + or its `job_type` does not match the desired subclass.
> + """
> + git_job = IStore(GitJob).get(GitJob, job_id)
> + if git_job.job_type != cls.class_job_type:
> + raise NotFoundError(
> + "No object found with id %d and type %s" %
> + (job_id, cls.class_job_type.title))
> + return cls(git_job)
> +
> + @classmethod
> + def iterReady(cls):
> + """See `IJobSource`."""
> + jobs = IMasterStore(GitJob).find(
> + GitJob,
> + GitJob.job_type == cls.class_job_type,
> + GitJob.job == Job.id,
> + Job.id.is_in(Job.ready_jobs))
> + return (cls(job) for job in jobs)
> +
> + def getOopsVars(self):
> + """See `IRunnableJob`."""
> + oops_vars = super(GitJobDerived, self).getOopsVars()
> + oops_vars.extend([
> + ('git_job_id', self.context.job.id),
> + ('git_job_type', self.context.job_type.title),
> + ('git_repository_id', self.context.repository.id),
> + ('git_repository_name', self.context.repository.unique_name)])
> + return oops_vars
> +
> + def getErrorRecipients(self):
> + if self.requester is None:
> + return []
> + return [format_address_for_person(self.requester)]
> +
> +
> +class GitRefScanJob(GitJobDerived):
> + """A Job that scans a Git repository for its current list of references."""
> +
> + implements(IGitRefScanJob)
> +
> + classProvides(IGitRefScanJobSource)
> + class_job_type = GitJobType.REF_SCAN
> +
> + max_retries = 5
> +
> + retry_error_types = (AdvisoryLockHeld,)
> +
> + config = config.IGitRefScanJobSource
> +
> + @classmethod
> + def create(cls, repository):
> + """See `IGitRefScanJobSource`."""
> + git_job = GitJob(
> + repository, cls.class_job_type,
> + {"repository_name": repository.unique_name})
> + job = cls(git_job)
> + job.celeryRunOnCommit()
> + return job
> +
> + def __init__(self, git_job):
> + super(GitRefScanJob, self).__init__(git_job)
> + self._cached_repository_name = self.metadata["repository_name"]
> + self._hosting_client = GitHostingClient(
> + config.codehosting.internal_git_api_endpoint)
> +
> + def run(self):
> + """See `IGitRefScanJob`."""
> + try:
> + with try_advisory_lock(
> + LockType.GIT_REF_SCAN, self.repository.id,
> + Store.of(self.repository)):
> + hosting_path = self.repository.getInternalPath()
> + self.repository.synchroniseRefs(
> + self._hosting_client.get_refs(hosting_path))
> + except LostObjectError:
> + log.warning(
> + "Skipping repository %s because it has been deleted." %
> + self._cached_repository_name)
>
> === modified file 'lib/lp/code/model/gitlookup.py'
> --- lib/lp/code/model/gitlookup.py 2015-03-05 11:39:06 +0000
> +++ lib/lp/code/model/gitlookup.py 2015-03-16 16:23:14 +0000
> @@ -297,6 +297,16 @@
> return default
> return repository
>
> + def getByHostingPath(self, path):
> + """See `IGitLookup`."""
> + # This may need to change later to improve support for sharding.
> + # See also `IGitRepository.getInternalPath`.
> + try:
> + repository_id = int(path)
> + except ValueError:
> + return None
> + return self.get(repository_id)
> +
> @staticmethod
> def uriToPath(uri):
> """See `IGitLookup`."""
>
> === added file 'lib/lp/code/model/gitref.py'
> --- lib/lp/code/model/gitref.py 1970-01-01 00:00:00 +0000
> +++ lib/lp/code/model/gitref.py 2015-03-16 16:23:14 +0000
> @@ -0,0 +1,37 @@
> +# Copyright 2015 Canonical Ltd. This software is licensed under the
> +# GNU Affero General Public License version 3 (see the file LICENSE).
> +
> +__metaclass__ = type
> +__all__ = [
> + 'GitRef',
> + ]
> +
> +from storm.locals import (
> + Int,
> + Reference,
> + Unicode,
> + )
> +from zope.interface import implements
> +
> +from lp.code.enums import GitObjectType
> +from lp.code.interfaces.gitref import IGitRef
> +from lp.services.database.enumcol import EnumCol
> +from lp.services.database.stormbase import StormBase
> +
> +
> +class GitRef(StormBase):
> + """See `IGitRef`."""
> +
> + __storm_table__ = 'GitRef'
> + __storm_primary__ = ('repository_id', 'path')
> +
> + implements(IGitRef)
> +
> + repository_id = Int(name='repository', allow_none=False)
> + repository = Reference(repository_id, 'GitRepository.id')
> +
> + path = Unicode(name='path', allow_none=False)
> +
> + commit_sha1 = Unicode(name='commit_sha1', allow_none=False)
> +
> + object_type = EnumCol(enum=GitObjectType, notNull=True)
>
> === modified file 'lib/lp/code/model/gitrepository.py'
> --- lib/lp/code/model/gitrepository.py 2015-03-05 14:13:16 +0000
> +++ lib/lp/code/model/gitrepository.py 2015-03-16 16:23:14 +0000
> @@ -8,15 +8,24 @@
> 'GitRepositorySet',
> ]
>
> +from itertools import chain
> +
> from bzrlib import urlutils
> import pytz
> +from storm.databases.postgres import Returning
> from storm.expr import (
> + And,
> Coalesce,
> + Insert,
> Join,
> Or,
> Select,
> SQL,
> )
> +from storm.info import (
> + ClassAlias,
> + get_cls_info,
> + )
> from storm.locals import (
> Bool,
> DateTime,
> @@ -24,6 +33,7 @@
> Reference,
> Unicode,
> )
> +from storm.store import Store
> from zope.component import getUtility
> from zope.interface import implements
> from zope.security.proxy import removeSecurityProxy
> @@ -36,6 +46,7 @@
> from lp.app.interfaces.informationtype import IInformationType
> from lp.app.interfaces.launchpad import IPrivacy
> from lp.app.interfaces.services import IService
> +from lp.code.enums import GitObjectType
> from lp.code.errors import (
> GitDefaultConflict,
> GitFeatureDisabled,
> @@ -57,6 +68,7 @@
> IGitRepositorySet,
> user_has_special_git_repository_access,
> )
> +from lp.code.model.gitref import GitRef
> from lp.registry.enums import PersonVisibility
> from lp.registry.errors import CannotChangeInformationType
> from lp.registry.interfaces.accesspolicy import (
> @@ -78,6 +90,7 @@
> )
> from lp.registry.model.teammembership import TeamParticipation
> from lp.services.config import config
> +from lp.services.database import bulk
> from lp.services.database.constants import (
> DEFAULT,
> UTC_NOW,
> @@ -89,12 +102,25 @@
> Array,
> ArrayAgg,
> ArrayIntersects,
> + BulkUpdate,
> + Values,
> )
> from lp.services.features import getFeatureFlag
> -from lp.services.propertycache import cachedproperty
> +from lp.services.propertycache import (
> + cachedproperty,
> + get_property_cache,
> + )
> from lp.services.webapp.authorization import available_with_permission
>
>
> +object_type_map = {
> + "commit": GitObjectType.COMMIT,
> + "tree": GitObjectType.TREE,
> + "blob": GitObjectType.BLOB,
> + "tag": GitObjectType.TAG,
> + }
> +
> +
> def git_repository_modified(repository, event):
> """Update the date_last_modified property when a GitRepository is modified.
>
> @@ -243,6 +269,7 @@
> def getInternalPath(self):
> """See `IGitRepository`."""
> # This may need to change later to improve support for sharding.
> + # See also `IGitLookup.getByHostingPath`.
> return str(self.id)
>
> def getCodebrowseUrl(self):
> @@ -279,6 +306,129 @@
> self, self.information_type, pillars, wanted_links)
>
> @cachedproperty
> + def refs(self):
> + """See `IGitRepository`."""
> + return list(Store.of(self).find(
> + GitRef, GitRef.repository_id == self.id).order_by(GitRef.path))
> +
> + def getRefByPath(self, path):
> + return Store.of(self).find(
> + GitRef,
> + GitRef.repository_id == self.id,
> + GitRef.path == path).one()
> +
> + @staticmethod
> + def _convertRefInfo(info):
> + """Validate and canonicalise ref info from the hosting service.
> +
> + :param info: A dict of {"object":
> + {"sha1": sha1, "type": "commit"/"tree"/"blob"/"tag"}}.
> +
> + :raises ValueError: if the dict is malformed.
> + :return: A dict of {"sha1": sha1, "type": `GitObjectType`}.
> + """
> + if "object" not in info:
> + raise ValueError('ref info does not contain "object" key')
> + obj = info["object"]
> + if "sha1" not in obj:
> + raise ValueError('ref info object does not contain "sha1" key')
> + if "type" not in obj:
> + raise ValueError('ref info object does not contain "type" key')
> + if not isinstance(obj["sha1"], basestring) or len(obj["sha1"]) != 40:
> + raise ValueError('ref info sha1 is not a 40-character string')
> + if obj["type"] not in object_type_map:
> + raise ValueError('ref info type is not a recognised object type')
> + sha1 = obj["sha1"]
> + if isinstance(sha1, bytes):
> + sha1 = sha1.decode("US-ASCII")
> + return {"sha1": sha1, "type": object_type_map[obj["type"]]}
> +
> + def createOrUpdateRefs(self, refs_info, get_objects=False):
> + """See `IGitRepository`."""
> + def dbify_values(values):
> + return [
> + list(chain.from_iterable(
> + bulk.dbify_value(col, val)
> + for col, val in zip(columns, value)))
> + for value in values]
> +
> + # Flush everything up to here, as we may need to invalidate the
> + # cache after updating.
> + store = Store.of(self)
> + store.flush()
> +
> + # Try a bulk update first.
> + column_names = ["repository_id", "path", "commit_sha1", "object_type"]
> + column_types = [
> + ("repository", "integer"),
> + ("path", "text"),
> + ("commit_sha1", "character(40)"),
> + ("object_type", "integer"),
> + ]
> + columns = [getattr(GitRef, name) for name in column_names]
> + values = [
> + (self.id, path, info["sha1"], info["type"])
> + for path, info in refs_info.items()]
> + db_values = dbify_values(values)
> + new_refs_expr = Values("new_refs", column_types, db_values)
> + new_refs = ClassAlias(GitRef, "new_refs")
> + updated_columns = dict([
> + (getattr(GitRef, name), getattr(new_refs, name))
> + for name in column_names if name not in ("repository_id", "path")])
> + update_filter = And(
> + GitRef.repository_id == new_refs.repository_id,
> + GitRef.path == new_refs.path)
> + primary_key = get_cls_info(GitRef).primary_key
> + updated = list(store.execute(Returning(BulkUpdate(
> + updated_columns, table=GitRef, values=new_refs_expr,
> + where=update_filter, primary_columns=primary_key))))
> + if updated:
> + # Some existing GitRef objects may no longer be valid. Without
> + # knowing which ones we already have, it's safest to just
> + # invalidate everything.
> + store.invalidate()
> +
> + # If there are any remaining items, create them.
> + create_db_values = dbify_values([
> + value for value in values if (value[0], value[1]) not in updated])
> + if create_db_values:
> + created = list(store.execute(Returning(Insert(
> + columns, values=create_db_values,
> + primary_columns=primary_key))))
> + else:
> + created = []
Yes, and also because I no longer want to pass get_objects through to it since I can load both updated and created GitRef objects in one query by avoiding that. Given that, bulk.create isn't doing much useful work that I haven't already had to do for the update, and so I might as well make the symmetries between update and create more obvious by inlining the two relevant statements.
> +
> + del get_property_cache(self).refs
> + if get_objects:
> + return bulk.load(GitRef, updated + created)
> +
> + def removeRefs(self, paths):
> + """See `IGitRepository`."""
> + Store.of(self).find(
> + GitRef,
> + GitRef.repository == self, GitRef.path.is_in(paths)).remove()
> + del get_property_cache(self).refs
> +
> + def synchroniseRefs(self, hosting_refs):
> + """See `IGitRepository`."""
> + new_refs = {}
> + for path, info in hosting_refs.items():
> + try:
> + new_refs[path] = self._convertRefInfo(info)
> + except ValueError:
> + pass
It might just be some kind of lack of mutual upgrade synchronisation in the future; I think it's worth trying to convert other refs as well rather than crashing on the first broken one. But you're right that it should at least be logged.
> + current_refs = dict((ref.path, ref) for ref in self.refs)
> + refs_to_upsert = {}
> + for path, info in new_refs.items():
> + current_ref = current_refs.get(path)
> + if (current_ref is None or
> + info["sha1"] != current_ref.commit_sha1 or
> + info["type"] != current_ref.object_type):
Indeed, I prefer to check all attributes for exactly that reason: if nothing else, we'll soon be adding tip commit information. I'll probably refactor this at that point to have neater comparison code, but that isn't quite worth it yet.
> + refs_to_upsert[path] = info
> + self.createOrUpdateRefs(refs_to_upsert)
> + self.removeRefs(set(current_refs) - set(new_refs))
> +
> + @cachedproperty
> def _known_viewers(self):
> """A set of known persons able to view this repository.
>
>
> === added file 'lib/lp/code/model/tests/test_gitjob.py'
> --- lib/lp/code/model/tests/test_gitjob.py 1970-01-01 00:00:00 +0000
> +++ lib/lp/code/model/tests/test_gitjob.py 2015-03-16 16:23:14 +0000
> @@ -0,0 +1,113 @@
> +# Copyright 2015 Canonical Ltd. This software is licensed under the
> +# GNU Affero General Public License version 3 (see the file LICENSE).
> +
> +"""Tests for `GitJob`s."""
> +
> +__metaclass__ = type
> +
> +import hashlib
> +
> +from testtools.matchers import (
> + MatchesSetwise,
> + MatchesStructure,
> + )
> +
> +from lp.code.enums import GitObjectType
> +from lp.code.interfaces.gitjob import (
> + IGitJob,
> + IGitRefScanJob,
> + )
> +from lp.code.interfaces.gitrepository import GIT_FEATURE_FLAG
> +from lp.code.model.gitjob import (
> + GitJob,
> + GitJobDerived,
> + GitJobType,
> + GitRefScanJob,
> + )
> +from lp.services.features.testing import FeatureFixture
> +from lp.testing import TestCaseWithFactory
> +from lp.testing.dbuser import dbuser
> +from lp.testing.fakemethod import FakeMethod
> +from lp.testing.layers import (
> + DatabaseFunctionalLayer,
> + LaunchpadZopelessLayer,
> + )
> +
> +
> +class TestGitJob(TestCaseWithFactory):
> + """Tests for `GitJob`."""
> +
> + layer = DatabaseFunctionalLayer
> +
> + def test_provides_interface(self):
> + # `GitJob` objects provide `IGitJob`.
> + self.useFixture(FeatureFixture({GIT_FEATURE_FLAG: u"on"}))
> + repository = self.factory.makeGitRepository()
> + self.assertProvides(
> + GitJob(repository, GitJobType.REF_SCAN, {}), IGitJob)
> +
> +
> +class TestGitJobDerived(TestCaseWithFactory):
> + """Tests for `GitJobDerived`."""
> +
> + layer = LaunchpadZopelessLayer
> +
> + def test_getOopsMailController(self):
> + """By default, no mail is sent about failed BranchJobs."""
> + self.useFixture(FeatureFixture({GIT_FEATURE_FLAG: u"on"}))
> + repository = self.factory.makeGitRepository()
> + job = GitJob(repository, GitJobType.REF_SCAN, {})
> + derived = GitJobDerived(job)
> + self.assertIsNone(derived.getOopsMailController("x"))
> +
> +
> +class TestGitRefScanJobMixin:
> +
> + @staticmethod
> + def makeFakeRefs(paths):
> + return dict(
> + (path, {"object": {
> + "sha1": hashlib.sha1(path).hexdigest(),
> + "type": "commit",
> + }})
> + for path in paths)
> +
> + def assertRefsMatch(self, refs, repository, paths):
> + matchers = [
> + MatchesStructure.byEquality(
> + repository=repository,
> + path=path,
> + commit_sha1=unicode(hashlib.sha1(path).hexdigest()),
> + object_type=GitObjectType.COMMIT)
> + for path in paths]
> + self.assertThat(refs, MatchesSetwise(*matchers))
> +
> +
> +class TestGitRefScanJob(TestGitRefScanJobMixin, TestCaseWithFactory):
> + """Tests for `GitRefScanJob`."""
> +
> + layer = LaunchpadZopelessLayer
> +
> + def setUp(self):
> + super(TestGitRefScanJob, self).setUp()
> + self.useFixture(FeatureFixture({GIT_FEATURE_FLAG: u"on"}))
> +
> + def test_provides_interface(self):
> + # `GitRefScanJob` objects provide `IGitRefScanJob`.
> + repository = self.factory.makeGitRepository()
> + self.assertProvides(GitRefScanJob.create(repository), IGitRefScanJob)
> +
> + def test_run(self):
> + # Ensure the job scans the repository.
> + repository = self.factory.makeGitRepository()
> + job = GitRefScanJob.create(repository)
> + paths = (u"refs/heads/master", u"refs/tags/1.0")
> + job._hosting_client.get_refs = FakeMethod(
> + result=self.makeFakeRefs(paths))
> + with dbuser("branchscanner"):
> + job.run()
> + self.assertRefsMatch(repository.refs, repository, paths)
> +
> +
> +# XXX cjwatson 2015-03-12: We should test that the job works via Celery too,
> +# but that isn't feasible until we have a proper turnip fixture.
>
> === modified file 'lib/lp/code/model/tests/test_gitlookup.py'
> --- lib/lp/code/model/tests/test_gitlookup.py 2015-03-05 14:13:16 +0000
> +++ lib/lp/code/model/tests/test_gitlookup.py 2015-03-16 16:23:14 +0000
> @@ -35,6 +35,26 @@
> from lp.testing.layers import DatabaseFunctionalLayer
>
>
> +class TestGetByHostingPath(TestCaseWithFactory):
> + """Test `IGitLookup.getByHostingPath`."""
> +
> + layer = DatabaseFunctionalLayer
> +
> + def setUp(self):
> + super(TestGetByHostingPath, self).setUp()
> + self.useFixture(FeatureFixture({GIT_FEATURE_FLAG: u"on"}))
> + self.lookup = getUtility(IGitLookup)
> +
> + def test_exists(self):
> + repository = self.factory.makeGitRepository()
> + self.assertEqual(
> + repository,
> + self.lookup.getByHostingPath(repository.getInternalPath()))
> +
> + def test_missing(self):
> + self.assertIsNone(self.lookup.getByHostingPath("nonexistent"))
> +
> +
> class TestGetByUniqueName(TestCaseWithFactory):
> """Tests for `IGitLookup.getByUniqueName`."""
>
>
> === modified file 'lib/lp/code/model/tests/test_gitrepository.py'
> --- lib/lp/code/model/tests/test_gitrepository.py 2015-03-06 16:31:30 +0000
> +++ lib/lp/code/model/tests/test_gitrepository.py 2015-03-16 16:23:14 +0000
> @@ -7,10 +7,15 @@
>
> from datetime import datetime
> from functools import partial
> +import hashlib
> import json
>
> from lazr.lifecycle.event import ObjectModifiedEvent
> import pytz
> +from testtools.matchers import (
> + MatchesSetwise,
> + MatchesStructure,
> + )
> from zope.component import getUtility
> from zope.event import notify
> from zope.security.proxy import removeSecurityProxy
> @@ -21,6 +26,7 @@
> PUBLIC_INFORMATION_TYPES,
> )
> from lp.app.interfaces.launchpad import ILaunchpadCelebrities
> +from lp.code.enums import GitObjectType
> from lp.code.errors import (
> GitFeatureDisabled,
> GitRepositoryCreatorNotMemberOfOwnerTeam,
> @@ -37,6 +43,7 @@
> IGitRepository,
> IGitRepositorySet,
> )
> +from lp.code.model.gitrepository import GitRepository
> from lp.registry.enums import (
> BranchSharingPolicy,
> PersonVisibility,
> @@ -410,6 +417,171 @@
> get_policies_for_artifact(repository))
>
>
> +class TestGitRepositoryRefs(TestCaseWithFactory):
> + """Tests for ref handling."""
> +
> + layer = DatabaseFunctionalLayer
> +
> + def setUp(self):
> + super(TestGitRepositoryRefs, self).setUp()
> + self.useFixture(FeatureFixture({GIT_FEATURE_FLAG: u"on"}))
> +
> + def test__convertRefInfo(self):
> + # _convertRefInfo converts a valid info dictionary.
> + sha1 = unicode(hashlib.sha1("").hexdigest())
> + info = {"object": {"sha1": sha1, "type": u"commit"}}
> + expected_info = {"sha1": sha1, "type": GitObjectType.COMMIT}
> + self.assertEqual(expected_info, GitRepository._convertRefInfo(info))
> +
> + def test__convertRefInfo_requires_object(self):
> + self.assertRaisesWithContent(
> + ValueError, 'ref info does not contain "object" key',
> + GitRepository._convertRefInfo, {})
> +
> + def test__convertRefInfo_requires_object_sha1(self):
> + self.assertRaisesWithContent(
> + ValueError, 'ref info object does not contain "sha1" key',
> + GitRepository._convertRefInfo, {"object": {}})
> +
> + def test__convertRefInfo_requires_object_type(self):
> + info = {
> + "object": {"sha1": u"0000000000000000000000000000000000000000"},
> + }
> + self.assertRaisesWithContent(
> + ValueError, 'ref info object does not contain "type" key',
> + GitRepository._convertRefInfo, info)
> +
> + def test__convertRefInfo_bad_sha1(self):
> + info = {"object": {"sha1": "x", "type": "commit"}}
> + self.assertRaisesWithContent(
> + ValueError, 'ref info sha1 is not a 40-character string',
> + GitRepository._convertRefInfo, info)
> +
> + def test__convertRefInfo_bad_type(self):
> + info = {
> + "object": {
> + "sha1": u"0000000000000000000000000000000000000000",
> + "type": u"nonsense",
> + },
> + }
> + self.assertRaisesWithContent(
> + ValueError, 'ref info type is not a recognised object type',
> + GitRepository._convertRefInfo, info)
> +
> + def assertRefsMatch(self, refs, repository, paths):
> + matchers = [
> + MatchesStructure.byEquality(
> + repository=repository,
> + path=path,
> + commit_sha1=unicode(hashlib.sha1(path).hexdigest()),
> + object_type=GitObjectType.COMMIT)
> + for path in paths]
> + self.assertThat(refs, MatchesSetwise(*matchers))
> +
> + def test_createRefs(self):
> + repository = self.factory.makeGitRepository()
> + self.assertEqual([], repository.refs)
> + paths = (u"refs/heads/master", u"refs/tags/1.0")
> + self.factory.makeGitRefs(repository=repository, paths=paths)
> + self.assertRefsMatch(repository.refs, repository, paths)
> + master_ref = repository.getRefByPath(u"refs/heads/master")
> + new_refs_info = {
> + u"refs/tags/1.1": {
> + u"sha1": master_ref.commit_sha1,
> + u"type": master_ref.object_type,
> + },
> + }
> + repository.createOrUpdateRefs(new_refs_info)
> + self.assertRefsMatch(
> + [ref for ref in repository.refs if ref.path != u"refs/tags/1.1"],
> + repository, paths)
> + self.assertThat(
> + repository.getRefByPath(u"refs/tags/1.1"),
> + MatchesStructure.byEquality(
> + repository=repository,
> + path=u"refs/tags/1.1",
> + commit_sha1=master_ref.commit_sha1,
> + object_type=master_ref.object_type,
> + ))
> +
> + def test_removeRefs(self):
> + repository = self.factory.makeGitRepository()
> + paths = (u"refs/heads/master", u"refs/heads/branch", u"refs/tags/1.0")
> + self.factory.makeGitRefs(repository=repository, paths=paths)
> + self.assertRefsMatch(repository.refs, repository, paths)
> + repository.removeRefs([u"refs/heads/branch", u"refs/tags/1.0"])
> + self.assertRefsMatch(
> + repository.refs, repository, [u"refs/heads/master"])
> +
> + def test_updateRef(self):
> + repository = self.factory.makeGitRepository()
> + paths = (u"refs/heads/master", u"refs/tags/1.0")
> + self.factory.makeGitRefs(repository=repository, paths=paths)
> + self.assertRefsMatch(repository.refs, repository, paths)
> + new_info = {
> + u"sha1": u"0000000000000000000000000000000000000000",
> + u"type": GitObjectType.BLOB,
> + }
> + repository.createOrUpdateRefs({u"refs/tags/1.0": new_info})
> + self.assertRefsMatch(
> + [ref for ref in repository.refs if ref.path != u"refs/tags/1.0"],
> + repository, [u"refs/heads/master"])
> + self.assertThat(
> + repository.getRefByPath(u"refs/tags/1.0"),
> + MatchesStructure.byEquality(
> + repository=repository,
> + path=u"refs/tags/1.0",
> + commit_sha1=u"0000000000000000000000000000000000000000",
> + object_type=GitObjectType.BLOB,
> + ))
> +
> + def test_synchroniseRefs(self):
> + # synchroniseRefs copes with synchronising a repository where some
> + # refs have been created, some deleted, and some changed.
> + repository = self.factory.makeGitRepository()
> + paths = (u"refs/heads/master", u"refs/heads/foo", u"refs/heads/bar")
> + self.factory.makeGitRefs(repository=repository, paths=paths)
> + self.assertRefsMatch(repository.refs, repository, paths)
> + repository.synchroniseRefs({
> + u"refs/heads/master": {
> + u"object": {
> + u"sha1": u"1111111111111111111111111111111111111111",
> + u"type": u"commit",
> + },
> + },
> + u"refs/heads/foo": {
> + u"object": {
> + u"sha1": repository.getRefByPath(
> + u"refs/heads/foo").commit_sha1,
> + u"type": u"commit",
> + },
> + },
> + u"refs/tags/1.0": {
> + u"object": {
> + u"sha1": repository.getRefByPath(
> + u"refs/heads/master").commit_sha1,
> + u"type": u"commit",
> + },
> + },
> + })
> + expected_sha1s = [
> + (u"refs/heads/master",
> + u"1111111111111111111111111111111111111111"),
> + (u"refs/heads/foo",
> + unicode(hashlib.sha1(u"refs/heads/foo").hexdigest())),
> + (u"refs/tags/1.0",
> + unicode(hashlib.sha1(u"refs/heads/master").hexdigest())),
> + ]
> + matchers = [
> + MatchesStructure.byEquality(
> + repository=repository,
> + path=path,
> + commit_sha1=sha1,
> + object_type=GitObjectType.COMMIT,
> + ) for path, sha1 in expected_sha1s]
> + self.assertThat(repository.refs, MatchesSetwise(*matchers))
> +
> +
> class TestGitRepositoryGetAllowedInformationTypes(TestCaseWithFactory):
> """Test `IGitRepository.getAllowedInformationTypes`."""
>
>
> === modified file 'lib/lp/code/xmlrpc/git.py'
> --- lib/lp/code/xmlrpc/git.py 2015-03-03 17:09:33 +0000
> +++ lib/lp/code/xmlrpc/git.py 2015-03-16 16:23:14 +0000
> @@ -38,6 +38,7 @@
> split_git_unique_name,
> )
> from lp.code.interfaces.gitrepository import IGitRepositorySet
> +from lp.code.interfaces.gitjob import IGitRefScanJobSource
> from lp.code.xmlrpc.codehosting import run_with_login
> from lp.registry.errors import (
> InvalidName,
> @@ -232,3 +233,12 @@
> return run_with_login(
> requester_id, self._translatePath,
> path.strip("/"), permission, can_authenticate)
> +
> + def notify(self, translated_path):
> + """See `IGitAPI`."""
> + repository = getUtility(IGitLookup).getByHostingPath(translated_path)
> + if repository is None:
> + return faults.NotFound(
> + "No repository found for '%s'." % translated_path)
> + job = getUtility(IGitRefScanJobSource).create(repository)
> + job.celeryRunOnCommit()
>
> === modified file 'lib/lp/code/xmlrpc/tests/test_git.py'
> --- lib/lp/code/xmlrpc/tests/test_git.py 2015-03-04 18:27:40 +0000
> +++ lib/lp/code/xmlrpc/tests/test_git.py 2015-03-16 16:23:14 +0000
> @@ -15,6 +15,7 @@
> LAUNCHPAD_SERVICES,
> )
> from lp.code.interfaces.gitcollection import IAllGitRepositories
> +from lp.code.interfaces.gitjob import IGitRefScanJobSource
> from lp.code.interfaces.gitrepository import (
> GIT_FEATURE_FLAG,
> GIT_REPOSITORY_NAME_VALIDATION_ERROR_MESSAGE,
> @@ -621,6 +622,22 @@
> "GitRepositoryCreationFault: nothing here",
> self.oopses[0]["tb_text"])
>
> + def test_notify(self):
> + # The notify call creates a GitRefScanJob.
> + repository = self.factory.makeGitRepository()
> + self.assertIsNone(self.git_api.notify(repository.getInternalPath()))
> + job_source = getUtility(IGitRefScanJobSource)
> + [job] = list(job_source.iterReady())
> + self.assertEqual(repository, job.repository)
> +
> + def test_notify_missing_repository(self):
> + # A notify call on a non-existent repository returns a fault and
> + # does not create a job.
> + fault = self.git_api.notify("10000")
> + self.assertIsInstance(fault, faults.NotFound)
> + job_source = getUtility(IGitRefScanJobSource)
> + self.assertEqual([], list(job_source.iterReady()))
> +
>
> class TestGitAPISecurity(TestGitAPIMixin, TestCaseWithFactory):
> """Slow tests for `IGitAPI`.
>
> === modified file 'lib/lp/scripts/garbo.py'
> --- lib/lp/scripts/garbo.py 2014-11-06 02:22:57 +0000
> +++ lib/lp/scripts/garbo.py 2015-03-16 16:23:14 +0000
> @@ -1,4 +1,4 @@
> -# Copyright 2009-2014 Canonical Ltd. This software is licensed under the
> +# Copyright 2009-2015 Canonical Ltd. This software is licensed under the
> # GNU Affero General Public License version 3 (see the file LICENSE).
>
> """Database garbage collection."""
> @@ -1092,6 +1092,23 @@
> """
>
>
> +class GitJobPruner(BulkPruner):
> + """Prune `GitJob`s that are in a final state and more than a month old.
> +
> + When a GitJob is completed, it gets set to a final state. These jobs
> + should be pruned from the database after a month.
> + """
> + target_table_class = Job
> + ids_to_prune_query = """
> + SELECT DISTINCT Job.id
> + FROM Job, GitJob
> + WHERE
> + Job.id = GitJob.job
> + AND Job.date_finished < CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
> + - CAST('30 days' AS interval)
> + """
> +
> +
> class BugHeatUpdater(TunableLoop):
> """A `TunableLoop` for bug heat calculations."""
>
> @@ -1645,6 +1662,7 @@
> BugWatchActivityPruner,
> CodeImportEventPruner,
> CodeImportResultPruner,
> + GitJobPruner,
> HWSubmissionEmailLinker,
> LiveFSFilePruner,
> LoginTokenPruner,
>
> === modified file 'lib/lp/scripts/tests/test_garbo.py'
> --- lib/lp/scripts/tests/test_garbo.py 2015-01-07 00:35:41 +0000
> +++ lib/lp/scripts/tests/test_garbo.py 2015-03-16 16:23:14 +0000
> @@ -1,4 +1,4 @@
> -# Copyright 2009-2014 Canonical Ltd. This software is licensed under the
> +# Copyright 2009-2015 Canonical Ltd. This software is licensed under the
> # GNU Affero General Public License version 3 (see the file LICENSE).
>
> """Test the database garbage collector."""
> @@ -49,6 +49,7 @@
> )
> from lp.code.enums import CodeImportResultStatus
> from lp.code.interfaces.codeimportevent import ICodeImportEventSet
> +from lp.code.interfaces.gitrepository import GIT_FEATURE_FLAG
> from lp.code.model.branchjob import (
> BranchJob,
> BranchUpgradeJob,
> @@ -56,6 +57,10 @@
> from lp.code.model.codeimportevent import CodeImportEvent
> from lp.code.model.codeimportresult import CodeImportResult
> from lp.code.model.diff import Diff
> +from lp.code.model.gitjob import (
> + GitJob,
> + GitRefScanJob,
> + )
> from lp.registry.enums import (
> BranchSharingPolicy,
> BugSharingPolicy,
> @@ -930,6 +935,48 @@
> switch_dbuser('testadmin')
> self.assertEqual(store.find(BranchJob).count(), 1)
>
> + def test_GitJobPruner(self):
> + # Garbo should remove jobs completed over 30 days ago.
> + self.useFixture(FeatureFixture({GIT_FEATURE_FLAG: u'on'}))
> + switch_dbuser('testadmin')
> + store = IMasterStore(Job)
> +
> + db_repository = self.factory.makeGitRepository()
> + Store.of(db_repository).flush()
> + git_job = GitRefScanJob.create(db_repository)
> + git_job.job.date_finished = THIRTY_DAYS_AGO
> +
> + self.assertEqual(
> + 1,
> + store.find(GitJob, GitJob.repository == db_repository.id).count())
> +
> + self.runDaily()
> +
> + switch_dbuser('testadmin')
> + self.assertEqual(
> + 0,
> + store.find(GitJob, GitJob.repository == db_repository.id).count())
> +
> + def test_GitJobPruner_doesnt_prune_recent_jobs(self):
> + # Check to make sure the garbo doesn't remove jobs that aren't more
> + # than thirty days old.
> + self.useFixture(FeatureFixture({GIT_FEATURE_FLAG: u'on'}))
> + switch_dbuser('testadmin')
> + store = IMasterStore(Job)
> +
> + db_repository = self.factory.makeGitRepository()
> +
> + git_job = GitRefScanJob.create(db_repository)
> + git_job.job.date_finished = THIRTY_DAYS_AGO
> +
> + db_repository2 = self.factory.makeGitRepository()
> + GitRefScanJob.create(db_repository2)
> +
> + self.runDaily()
> +
> + switch_dbuser('testadmin')
> + self.assertEqual(1, store.find(GitJob).count())
> +
> def test_ObsoleteBugAttachmentPruner(self):
> # Bug attachments without a LibraryFileContent record are removed.
>
>
> === modified file 'lib/lp/security.py'
> --- lib/lp/security.py 2015-03-06 10:22:08 +0000
> +++ lib/lp/security.py 2015-03-16 16:23:14 +0000
> @@ -84,6 +84,7 @@
> from lp.code.interfaces.codereviewvote import ICodeReviewVoteReference
> from lp.code.interfaces.diff import IPreviewDiff
> from lp.code.interfaces.gitcollection import IGitCollection
> +from lp.code.interfaces.gitref import IGitRef
> from lp.code.interfaces.gitrepository import (
> IGitRepository,
> user_has_special_git_repository_access,
> @@ -2260,6 +2261,15 @@
> usedfor = IGitRepository
>
>
> +class ViewGitRef(DelegatedAuthorization):
> + """Anyone who can see a Git repository can see references within it."""
> + permission = 'launchpad.View'
> + usedfor = IGitRef
> +
> + def __init__(self, obj):
> + super(ViewGitRef, self).__init__(obj, obj.repository)
> +
> +
> class AdminDistroSeriesTranslations(AuthorizationBase):
> permission = 'launchpad.TranslationsAdmin'
> usedfor = IDistroSeries
>
> === modified file 'lib/lp/services/config/schema-lazr.conf'
> --- lib/lp/services/config/schema-lazr.conf 2015-02-19 17:33:35 +0000
> +++ lib/lp/services/config/schema-lazr.conf 2015-03-16 16:23:14 +0000
> @@ -1750,6 +1750,10 @@
> module: lp.soyuz.interfaces.distributionjob
> dbuser: distroseriesdifferencejob
>
> +[IGitRefScanJobSource]
> +module: lp.code.interfaces.gitjob
> +dbuser: branchscanner
> +
> [IInitializeDistroSeriesJobSource]
> module: lp.soyuz.interfaces.distributionjob
> dbuser: initializedistroseries
>
> === modified file 'lib/lp/services/database/locking.py'
> --- lib/lp/services/database/locking.py 2012-06-14 05:31:23 +0000
> +++ lib/lp/services/database/locking.py 2015-03-16 16:23:14 +0000
> @@ -1,4 +1,4 @@
> -# Copyright 2011-2012 Canonical Ltd. This software is licensed under the
> +# Copyright 2011-2015 Canonical Ltd. This software is licensed under the
> # GNU Affero General Public License version 3 (see the file LICENSE).
>
> __metaclass__ = type
> @@ -34,6 +34,11 @@
> Branch scan.
> """)
>
> + GIT_REF_SCAN = DBItem(1, """Git repository reference scan.
> +
> + Git repository reference scan.
> + """)
> +
>
> @contextmanager
> def try_advisory_lock(lock_type, lock_id, store):
>
> === modified file 'lib/lp/services/database/stormexpr.py'
> --- lib/lp/services/database/stormexpr.py 2013-05-02 22:22:16 +0000
> +++ lib/lp/services/database/stormexpr.py 2015-03-16 16:23:14 +0000
> @@ -1,4 +1,4 @@
> -# Copyright 2011 Canonical Ltd. This software is licensed under the
> +# Copyright 2011-2015 Canonical Ltd. This software is licensed under the
> # GNU Affero General Public License version 3 (see the file LICENSE).
>
> __metaclass__ = type
> @@ -47,13 +47,14 @@
>
> class BulkUpdate(Expr):
> # Perform a bulk table update using literal values.
> - __slots__ = ("map", "where", "table", "values")
> + __slots__ = ("map", "where", "table", "values", "primary_columns")
>
> - def __init__(self, map, table, values, where=Undef):
> + def __init__(self, map, table, values, where=Undef, primary_columns=Undef):
> self.map = map
> self.where = where
> self.table = table
> self.values = values
> + self.primary_columns = primary_columns
>
>
> @compile.when(BulkUpdate)
>
> === modified file 'lib/lp/testing/factory.py'
> --- lib/lp/testing/factory.py 2015-02-20 00:56:57 +0000
> +++ lib/lp/testing/factory.py 2015-03-16 16:23:14 +0000
> @@ -108,6 +108,7 @@
> CodeImportResultStatus,
> CodeImportReviewStatus,
> CodeReviewNotificationLevel,
> + GitObjectType,
> RevisionControlSystems,
> )
> from lp.code.errors import UnknownBranchTypeError
> @@ -1700,6 +1701,20 @@
> information_type, registrant, verify_policy=False)
> return repository
>
> + def makeGitRefs(self, repository=None, paths=None):
> + """Create and return a list of new, arbitrary GitRefs."""
> + if repository is None:
> + repository = self.makeGitRepository()
> + if paths is None:
> + paths = [self.getUniqueString('refs/heads/path').decode('utf-8')]
> + refs_info = dict(
> + (path, {
> + u"sha1": unicode(hashlib.sha1(path).hexdigest()),
> + u"type": GitObjectType.COMMIT,
> + })
> + for path in paths)
> + return repository.createOrUpdateRefs(refs_info, get_objects=True)
> +
> def makeBug(self, target=None, owner=None, bug_watch_url=None,
> information_type=None, date_closed=None, title=None,
> date_created=None, description=None, comment=None,
>
--
https://code.launchpad.net/~cjwatson/launchpad/git-ref-scanner/+merge/252763
Your team Launchpad code reviewers is subscribed to branch lp:launchpad.
References
