launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #03335
[Merge] lp:~stevenk/launchpad/drop-psc into lp:launchpad
Steve Kowalik has proposed merging lp:~stevenk/launchpad/drop-psc into lp:launchpad.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~stevenk/launchpad/drop-psc/+merge/57807
After running for one month, and processing the changelogs for as-near-as-makes-no-difference half a million SPRs, the PopulateSPRChangelogs class in the hourly garbo script has done its dash. So to thank it, we kick it out of the code base.
--
https://code.launchpad.net/~stevenk/launchpad/drop-psc/+merge/57807
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~stevenk/launchpad/drop-psc into lp:launchpad.
=== modified file 'database/schema/security.cfg'
--- database/schema/security.cfg 2011-04-06 15:35:05 +0000
+++ database/schema/security.cfg 2011-04-15 05:03:31 +0000
@@ -2260,9 +2260,6 @@
public.job = SELECT, INSERT, DELETE
public.branchjob = SELECT, DELETE
public.bugjob = SELECT, INSERT
-public.libraryfilealias = SELECT, INSERT
-public.libraryfilecontent = SELECT, INSERT
-public.sourcepackagerelease = SELECT, UPDATE
[garbo_daily]
type=user
=== modified file 'lib/lp/scripts/garbo.py'
--- lib/lp/scripts/garbo.py 2011-04-12 08:58:38 +0000
+++ lib/lp/scripts/garbo.py 2011-04-15 05:03:31 +0000
@@ -13,21 +13,14 @@
datetime,
timedelta,
)
-from fixtures import TempDir
import logging
import multiprocessing
-import os
-import signal
-import subprocess
import threading
import time
from psycopg2 import IntegrityError
import pytz
-from storm.expr import LeftJoin
from storm.locals import (
- And,
- Count,
Max,
Min,
SQL,
@@ -51,7 +44,6 @@
from canonical.launchpad.database.oauth import OAuthNonce
from canonical.launchpad.database.openidconsumer import OpenIDConsumerNonce
from canonical.launchpad.interfaces.emailaddress import EmailAddressStatus
-from canonical.launchpad.interfaces.librarian import ILibraryFileAliasSet
from canonical.launchpad.interfaces.lpstorm import IMasterStore
from canonical.launchpad.utilities.looptuner import TunableLoop
from canonical.launchpad.webapp.interfaces import (
@@ -59,9 +51,6 @@
MAIN_STORE,
MASTER_FLAVOR,
)
-from canonical.librarian.utils import copy_and_close
-from lp.archiveuploader.dscfile import findFile
-from lp.archiveuploader.nascentuploadfile import UploadError
from lp.bugs.interfaces.bug import IBugSet
from lp.bugs.model.bug import Bug
from lp.bugs.model.bugattachment import BugAttachment
@@ -82,14 +71,11 @@
from lp.registry.model.person import Person
from lp.services.job.model.job import Job
from lp.services.log.logger import PrefixFilter
-from lp.services.memcache.interfaces import IMemcacheClient
from lp.services.scripts.base import (
LaunchpadCronScript,
SilentLaunchpadScriptFailure,
)
from lp.services.session.model import SessionData
-from lp.soyuz.model.files import SourcePackageReleaseFile
-from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
from lp.translations.interfaces.potemplate import IPOTemplateSet
from lp.translations.model.potranslation import POTranslation
@@ -97,10 +83,6 @@
ONE_DAY_IN_SECONDS = 24*60*60
-def subprocess_setup():
- signal.signal(signal.SIGPIPE, signal.SIG_DFL)
-
-
class BulkPruner(TunableLoop):
"""A abstract ITunableLoop base class for simple pruners.
@@ -839,124 +821,6 @@
self.done = True
-class PopulateSPRChangelogs(TunableLoop):
- maximum_chunk_size = 1
-
- def __init__(self, log, abort_time=None):
- super(PopulateSPRChangelogs, self).__init__(log, abort_time)
- value = getUtility(IMemcacheClient).get('populate-spr-changelogs')
- if not value:
- self.start_at = 0
- else:
- self.start_at = value
- self.finish_at = self.getCandidateSPRs(0).last()
-
- def getCandidateSPRs(self, start_at):
- return IMasterStore(SourcePackageRelease).using(
- SourcePackageRelease,
- # Find any SPRFs that have expired (LFA.content IS NULL).
- LeftJoin(
- SourcePackageReleaseFile,
- SourcePackageReleaseFile.sourcepackagereleaseID ==
- SourcePackageRelease.id),
- LeftJoin(
- LibraryFileAlias,
- And(LibraryFileAlias.id ==
- SourcePackageReleaseFile.libraryfileID,
- LibraryFileAlias.content == None)),
- # And exclude any SPRs that have any expired SPRFs.
- ).find(
- SourcePackageRelease.id,
- SourcePackageRelease.id >= start_at,
- SourcePackageRelease.changelog == None,
- ).group_by(SourcePackageRelease.id).having(
- Count(LibraryFileAlias) == 0
- ).order_by(SourcePackageRelease.id)
-
- def isDone(self):
- return self.start_at > self.finish_at
-
- def __call__(self, chunk_size):
- for sprid in self.getCandidateSPRs(self.start_at)[:chunk_size]:
- spr = SourcePackageRelease.get(sprid)
- with TempDir() as tmp_dir:
- dsc_file = None
-
- # Grab the files from the librarian into a temporary
- # directory.
- try:
- for sprf in spr.files:
- dest = os.path.join(
- tmp_dir.path, sprf.libraryfile.filename)
- dest_file = open(dest, 'w')
- sprf.libraryfile.open()
- copy_and_close(sprf.libraryfile, dest_file)
- if dest.endswith('.dsc'):
- dsc_file = dest
- except LookupError:
- self.log.warning(
- 'SPR %d (%s %s) has missing library files.' % (
- spr.id, spr.name, spr.version))
- continue
-
- if dsc_file is None:
- self.log.warning(
- 'SPR %d (%s %s) has no DSC.' % (
- spr.id, spr.name, spr.version))
- continue
-
- # Extract the source package. Throw away stdout/stderr
- # -- we only really care about the return code.
- fnull = open('/dev/null', 'w')
- ret = subprocess.call(
- ['dpkg-source', '-x', dsc_file, os.path.join(
- tmp_dir.path, 'extracted')],
- stdout=fnull, stderr=fnull,
- preexec_fn=subprocess_setup)
- fnull.close()
- if ret != 0:
- self.log.warning(
- 'SPR %d (%s %s) failed to unpack: returned %d' % (
- spr.id, spr.name, spr.version, ret))
- continue
-
- # We have an extracted source package. Let's get the
- # changelog. findFile ensures that it's not too huge, and
- # not a symlink.
- try:
- changelog_path = findFile(
- tmp_dir.path, 'debian/changelog')
- except UploadError, e:
- changelog_path = None
- self.log.warning(
- 'SPR %d (%s %s) changelog could not be '
- 'imported: %s' % (
- spr.id, spr.name, spr.version, e))
- if changelog_path:
- # The LFA should be restricted only if there aren't any
- # public publications.
- restricted = not any(
- not a.private for a in spr.published_archives)
- spr.changelog = getUtility(ILibraryFileAliasSet).create(
- 'changelog',
- os.stat(changelog_path).st_size,
- open(changelog_path, "r"),
- "text/x-debian-source-changelog",
- restricted=restricted)
- self.log.info('SPR %d (%s %s) changelog imported.' % (
- spr.id, spr.name, spr.version))
- else:
- self.log.warning('SPR %d (%s %s) had no changelog.' % (
- spr.id, spr.name, spr.version))
-
- self.start_at = spr.id + 1
- result = getUtility(IMemcacheClient).set(
- 'populate-spr-changelogs', self.start_at)
- if not result:
- self.log.warning('Failed to set start_at in memcache.')
- transaction.commit()
-
-
class BaseDatabaseGarbageCollector(LaunchpadCronScript):
"""Abstract base class to run a collection of TunableLoops."""
script_name = None # Script name for locking and database user. Override.
@@ -1122,7 +986,6 @@
UnusedSessionPruner,
DuplicateSessionPruner,
BugHeatUpdater,
- PopulateSPRChangelogs,
]
experimental_tunable_loops = []
=== modified file 'lib/lp/scripts/tests/test_garbo.py'
--- lib/lp/scripts/tests/test_garbo.py 2011-04-08 14:10:32 +0000
+++ lib/lp/scripts/tests/test_garbo.py 2011-04-15 05:03:31 +0000
@@ -10,11 +10,8 @@
datetime,
timedelta,
)
-from fixtures import TempDir
import logging
-import os
from StringIO import StringIO
-import subprocess
import time
from pytz import UTC
@@ -42,7 +39,6 @@
from canonical.launchpad.database.oauth import OAuthNonce
from canonical.launchpad.database.openidconsumer import OpenIDConsumerNonce
from canonical.launchpad.interfaces.emailaddress import EmailAddressStatus
-from canonical.launchpad.interfaces.librarian import ILibraryFileAliasSet
from canonical.launchpad.interfaces.lpstorm import IMasterStore
from canonical.launchpad.scripts.tests import run_script
from canonical.launchpad.webapp.interfaces import (
@@ -56,7 +52,6 @@
LaunchpadZopelessLayer,
ZopelessDatabaseLayer,
)
-from lp.archiveuploader.dscfile import findFile
from lp.bugs.model.bugnotification import (
BugNotification,
BugNotificationRecipient,
@@ -73,7 +68,6 @@
)
from lp.code.model.codeimportevent import CodeImportEvent
from lp.code.model.codeimportresult import CodeImportResult
-from lp.registry.interfaces.distribution import IDistributionSet
from lp.registry.interfaces.person import (
IPersonSet,
PersonCreationRationale,
@@ -93,8 +87,6 @@
SessionData,
SessionPkgData,
)
-from lp.soyuz.enums import PackagePublishingStatus
-from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
from lp.testing import (
TestCase,
TestCaseWithFactory,
@@ -114,13 +106,6 @@
def test_hourly_script(self):
"""Ensure garbo-hourly.py actually runs."""
- # Our sampledata doesn't contain anything that PopulateSPRChangelogs
- # can process without errors, so it's easier to just set all of the
- # changelogs to a random LFA. We can't just expire every LFA, since
- # a bunch of SPRs have no SPRFs at all.
- IMasterStore(SourcePackageRelease).find(SourcePackageRelease).set(
- changelogID=1)
- transaction.commit() # run_script() is a different process.
rv, out, err = run_script(
"cronscripts/garbo-hourly.py", ["-q"], expect_returncode=0)
self.failIf(out.strip(), "Output to stdout: %s" % out)
@@ -889,60 +874,3 @@
""" % sqlbase.quote(template.id)).get_one()
self.assertEqual(1, count)
-
- def upload_to_debian(self, restricted=False):
- sid = getUtility(IDistributionSet)['debian']['sid']
- spn = self.factory.makeSourcePackageName('9wm')
- spr = self.factory.makeSourcePackageRelease(
- sourcepackagename=spn, version='1.2-7', distroseries=sid)
- archive = sid.main_archive
- if restricted:
- archive = self.factory.makeArchive(
- distribution=sid.distribution, private=True)
- self.factory.makeSourcePackagePublishingHistory(
- sourcepackagerelease=spr, archive=archive,
- status=PackagePublishingStatus.PUBLISHED)
- for name in (
- '9wm_1.2-7.diff.gz', '9wm_1.2.orig.tar.gz', '9wm_1.2-7.dsc'):
- path = os.path.join(
- 'lib/lp/soyuz/scripts/tests/gina_test_archive/pool/main/9',
- '9wm', name)
- lfa = getUtility(ILibraryFileAliasSet).create(
- name, os.stat(path).st_size, open(path, 'r'),
- 'application/octet-stream', restricted=restricted)
- spr.addFile(lfa)
- with TempDir() as tmp_dir:
- fnull = open('/dev/null', 'w')
- ret = subprocess.call(
- ['dpkg-source', '-x', path, os.path.join(
- tmp_dir.path, 'extracted')],
- stdout=fnull, stderr=fnull)
- fnull.close()
- self.assertEqual(0, ret)
- changelog_path = findFile(tmp_dir.path, 'debian/changelog')
- changelog = open(changelog_path, 'r').read()
- transaction.commit() # .runHourly() switches dbuser.
- return (spr, changelog)
-
- def test_populateSPRChangelogs(self):
- # We set SPR.changelog for imported records from Debian.
- LaunchpadZopelessLayer.switchDbUser('testadmin')
- spr, changelog = self.upload_to_debian()
- collector = self.runHourly()
- log = self.log_buffer.getvalue()
- self.assertTrue(
- 'SPR %d (9wm 1.2-7) changelog imported.' % spr.id in log)
- self.assertFalse(spr.changelog == None)
- self.assertFalse(spr.changelog.restricted)
- self.assertEqual(changelog, spr.changelog.read())
-
- def test_populateSPRChangelogs_restricted_sprf(self):
- LaunchpadZopelessLayer.switchDbUser('testadmin')
- spr, changelog = self.upload_to_debian(restricted=True)
- collector = self.runHourly()
- log = self.log_buffer.getvalue()
- self.assertTrue(
- 'SPR %d (9wm 1.2-7) changelog imported.' % spr.id in log)
- self.assertFalse(spr.changelog == None)
- self.assertTrue(spr.changelog.restricted)
- self.assertEqual(changelog, spr.changelog.read())
Follow ups
