launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #01930
[Merge] lp:~jtv/launchpad/pre-675426 into lp:launchpad/devel
Jeroen T. Vermeulen has proposed merging lp:~jtv/launchpad/pre-675426 into lp:launchpad/devel.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers): code
= Remove dead Translations code =
This gets rid of some dead, untested, obsolete Translations script code that we're only carrying around out of historical interest. We have bzr for historical interests.
Jeroen
--
https://code.launchpad.net/~jtv/launchpad/pre-675426/+merge/40953
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~jtv/launchpad/pre-675426 into lp:launchpad/devel.
=== modified file 'lib/lp/services/scripts/tests/__init__.py'
--- lib/lp/services/scripts/tests/__init__.py 2010-08-20 20:31:18 +0000
+++ lib/lp/services/scripts/tests/__init__.py 2010-11-16 13:00:24 +0000
@@ -30,7 +30,6 @@
'scripts/clean-sourceforge-project-entries.py',
'scripts/import-zope-specs.py',
'scripts/rosetta/gettext_check_messages.py',
- 'scripts/rosetta/remove-obsolete-translations.py',
# sqlobject.DatbaseIndex ?
'scripts/linkreport.py',
# Python executable without '.py' extension.
@@ -38,7 +37,6 @@
'scripts/queue',
# Bad script, no help.
'scripts/librarian-report.py',
- 'scripts/rosetta/message-sharing-populate-test.py',
'scripts/get-stacked-on-branches.py',
'scripts/start-loggerhead.py',
'scripts/stop-loggerhead.py',
=== removed file 'lib/lp/translations/scripts/migrate_kde_potemplates.py'
--- lib/lp/translations/scripts/migrate_kde_potemplates.py 2010-08-20 20:31:18 +0000
+++ lib/lp/translations/scripts/migrate_kde_potemplates.py 1970-01-01 00:00:00 +0000
@@ -1,339 +0,0 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-"""Migrate KDE POTemplates to native support for plural forms and context ."""
-
-__metaclass__ = type
-
-__all__ = [
- 'migrate_potemplates',
- ]
-
-from sqlobject import SQLObjectNotFound
-from zope.security.proxy import removeSecurityProxy
-
-from canonical.database.sqlbase import (
- cursor,
- sqlvalues,
- )
-from lp.translations.interfaces.translationfileformat import (
- TranslationFileFormat,
- )
-from lp.translations.interfaces.translations import TranslationConstants
-from lp.translations.model.pomsgid import POMsgID
-from lp.translations.model.potemplate import POTemplate
-from lp.translations.model.potmsgset import POTMsgSet
-from lp.translations.model.potranslation import POTranslation
-from lp.translations.model.translationmessage import TranslationMessage
-
-
-def getOrCreatePOMsgID(msgid):
- try:
- pomsgid = POMsgID.byMsgid(msgid)
- except SQLObjectNotFound:
- pomsgid = POMsgID(msgid=msgid)
- return pomsgid
-
-def get_potranslations(translations):
- potranslations = {}
- for index, translation in enumerate(translations):
- if translation != '':
- potranslations[index] = (
- POTranslation.getOrCreateTranslation(translation))
- else:
- potranslations[index] = None
- for index in range(len(potranslations),
- TranslationConstants.MAX_PLURAL_FORMS):
- potranslations[index] = None
- return potranslations
-
-
-def find_existing_translation(potmsgset, pofile, potranslations):
- unprotected_potmsgset = removeSecurityProxy(potmsgset)
- existing_message = (
- unprotected_potmsgset._findTranslationMessage(
- pofile=pofile, potranslations=potranslations,
- pluralforms=TranslationConstants.MAX_PLURAL_FORMS))
- return existing_message
-
-
-def migrate_translations_for_potmsgset(potmsgset, from_potmsgset,
- logger, ztm):
- """Migrate translations from `from_potmsgset` to `potmsgset`.
-
- `from_potmsgset` is an old, unmigrated POTMsgSet we are migrating
- translations from. Translations are migrated to native context
- and plural forms support along the way.
-
- `from_potmsgset` and `potmsgset` might be the same, which happens when
- we are migrating templates as well.
- """
- messages = TranslationMessage.select(
- "potmsgset = %s" % sqlvalues(from_potmsgset))
- logger.debug("Migrating %d translations for '%s'..." % (
- messages.count(), potmsgset.singular_text))
- for message in messages:
- msgstrs = message.translations
- # Let's see if translations have only the first plural
- # form defined: if they do, then they need migration.
- single_string = False
- if len(msgstrs) > 0 and msgstrs[0] is not None:
- single_string = True
- for msgstr in msgstrs[1:]:
- if msgstr is not None:
- single_string = False
-
- if single_string:
- translations = msgstrs[0].split('\n')
-
- # If there's only a single plural form, no need to change
- # anything. If POTMsgSets are different, we still need
- # to move the translation from one to the other.
- if len(translations) == 1 and potmsgset == from_potmsgset:
- continue
-
- # If there is an existing TranslationMessage with
- # these translations, re-use that and remove this one,
- # otherwise modify this one in-place.
- potranslations = get_potranslations(translations)
- existing_message = find_existing_translation(
- potmsgset, message.pofile, potranslations)
- if existing_message:
- if existing_message.id != message.id:
- # Only transfer is_current and is_imported
- # properties to an existing translation.
- if message.is_current:
- existing_message.is_current = True
- if message.is_imported:
- existing_message.is_imported = True
- # And remove the current message.
- message.destroySelf()
- else:
- # Modify `message` in-place.
- message.msgstr0 = potranslations[0]
- message.msgstr1 = potranslations[1]
- message.msgstr2 = potranslations[2]
- message.msgstr3 = potranslations[3]
- message.msgstr4 = potranslations[4]
- message.msgstr5 = potranslations[5]
- if potmsgset.id != from_potmsgset.id:
- # Point TranslationMessage to a new POTMsgSet.
- # To avoid hitting constraints, first unset
- # the is_current and is_imported flags, and
- # restore them afterwards.
- stored_is_current = message.is_current
- stored_is_imported = message.is_imported
- message.is_current = False
- message.is_imported = False
- message.potmsgset = potmsgset
- message.sync()
- message.is_current = stored_is_current
- message.is_imported = stored_is_imported
- message.sync()
-
-
-def migrate_kde_potemplate_translations(potemplate, logger, ztm):
- assert potemplate.source_file_format == TranslationFileFormat.KDEPO, (
- "Trying to move translations for non-KDEPO template.")
- cur = cursor()
- cur.execute("""
- SELECT old_msg.id, new_msg.id
- FROM POTMsgSet AS old_msg, POMsgID AS old_msgid,
- POTMsgSet AS new_msg, POMsgID AS singular, POMsgID AS plural
- WHERE
- -- they are both from this template
- old_msg.potemplate=%s AND
- new_msg.potemplate=old_msg.potemplate AND
- -- old one is obsolete
- old_msg.sequence=0 AND
- -- old POTMsgSet has a singular form of the form '_n:...',
- -- and no plural form
- old_msg.msgid_singular=old_msgid.id AND
- old_msg.msgid_plural IS NULL AND
- old_msgid.msgid LIKE E'\\\\_n: %%' AND
- -- and new POTMsgSet has singular and plural which when joined
- -- give the old plural form
- new_msg.msgid_singular=singular.id AND
- new_msg.msgid_plural=plural.id AND
- '_n: ' || singular.msgid || E'\\n' || plural.msgid = old_msgid.msgid
- """ % sqlvalues(potemplate))
- plural_potmsgsets = cur.fetchall()
-
- logger.info("Migrating translations for %d plural POTMsgSets..." % (
- len(plural_potmsgsets)))
- for old_potmsgset_id, new_potmsgset_id in plural_potmsgsets:
- old_potmsgset = POTMsgSet.get(old_potmsgset_id)
- new_potmsgset = POTMsgSet.get(new_potmsgset_id)
- migrate_translations_for_potmsgset(new_potmsgset, old_potmsgset,
- logger, ztm)
-
- cur.execute("""
- SELECT old_msg.id, new_msg.id
- FROM POTMsgSet AS old_msg, POMsgID AS old_msgid,
- POTMsgSet AS new_msg, POMsgID AS new_msgid
- WHERE
- -- they are both from this template
- old_msg.potemplate=%s AND
- new_msg.potemplate=old_msg.potemplate AND
- -- old one is obsolete
- old_msg.sequence=0 AND
- -- old POTMsgSet has a singular form of the form '_:...',
- -- and no plural form
- old_msg.msgid_singular=old_msgid.id AND
- old_msg.msgid_plural IS NULL AND
- old_msgid.msgid LIKE E'\\\\_: %%' AND
- -- and new POTMsgSet has singular and context which when joined
- -- give the old contextual message
- new_msg.msgid_singular=new_msgid.id AND
- '_: ' || new_msg.context || E'\\n' || new_msgid.msgid = old_msgid.msgid
- """ % sqlvalues(potemplate))
-
- plural_potmsgsets = cur.fetchall()
-
- logger.info("Migrating translations for %d context POTMsgSets..." % (
- len(plural_potmsgsets)))
- for old_potmsgset_id, new_potmsgset_id in plural_potmsgsets:
- old_potmsgset = POTMsgSet.get(old_potmsgset_id)
- new_potmsgset = POTMsgSet.get(new_potmsgset_id)
- messages = TranslationMessage.select(
- "potmsgset=%s" % sqlvalues(old_potmsgset))
- logger.debug(
- "Moving %d context translations from POTMsgSet %d to %d..." % (
- messages.count(), old_potmsgset_id, new_potmsgset_id))
- migrate_translations_for_potmsgset(new_potmsgset, old_potmsgset,
- logger, ztm)
-
- ztm.commit()
-
-
-def existing_potmsgset(potemplate, msgid_singular, context):
- clauses = ['potemplate=%s' % sqlvalues(potemplate),
- 'msgid_singular=%s' % sqlvalues(msgid_singular)]
- if context is None:
- clauses.append("context IS NULL")
- else:
- clauses.append("context=%s" % sqlvalues(context))
-
- return POTMsgSet.selectOne(" AND ".join(clauses))
-
-
-def migrate_potemplate(potemplate, logger, ztm):
- """Migrate PO templates to KDEPO format if needed."""
-
- plural_prefix = u'_n: '
- context_prefix = u'_: '
-
- assert(potemplate.source_file_format == TranslationFileFormat.PO)
-
- potmsgsets = POTMsgSet.select("""
- POTMsgSet.potemplate = %s AND
- POTMsgSet.msgid_singular=POMsgID.id AND
- POTMsgSet.msgid_plural IS NULL AND
- (POMsgID.msgid LIKE E'\\\\_n: %%' OR POMsgID.msgid LIKE E'\\\\_: %%')
- """ % sqlvalues(potemplate),
- clauseTables=['POMsgID'])
-
- logger.debug("Fixing %d POTMsgSets..." % potmsgsets.count())
-
- # We go potmsgset by potmsgset because it's easier to preserve
- # correctness. It'd be faster to go through translation messages
- # but then we'd be fixing potmsgsets after fixing translation messages.
- for potmsgset in potmsgsets:
- msgid = potmsgset.singular_text
- fix_plurals = fix_context = False
-
- # Detect plural form and context messages: use the same
- # logic as in utilities/kde_po_importer.py.
- if msgid.startswith(plural_prefix) and '\n' in msgid:
- # This is a KDE plural form.
- singular_text, plural_text = (
- msgid[len(plural_prefix):].split('\n', 1))
-
- msgid_singular = getOrCreatePOMsgID(singular_text)
-
- if existing_potmsgset(potemplate, msgid_singular, None):
- logger.warn("POTMsgSet %d conflicts with another one." % (
- potmsgset.id))
- else:
- potmsgset.msgid_singular = msgid_singular
- potmsgset.msgid_plural = getOrCreatePOMsgID(plural_text)
- potmsgset.sync()
- fix_plurals = True
- elif msgid.startswith(context_prefix) and '\n' in msgid:
- # This is a KDE context message: it needs no fixing apart
- # from changing msgid_singular.
- context, singular_text = (
- msgid[len(context_prefix):].split('\n', 1))
- msgid_singular = getOrCreatePOMsgID(singular_text)
-
- if existing_potmsgset(potemplate, msgid_singular, context):
- logger.warn("POTMsgSet %d conflicts with another one." % (
- potmsgset.id))
- else:
- potmsgset.context = context
- potmsgset.msgid_singular = msgid_singular
- potmsgset.sync()
- fix_context = True
- else:
- # Other messages here are the ones which begin like
- # context or plural form messages, but are actually neither.
- pass
-
- if fix_plurals:
- # Fix translations for this potmsgset.
- migrate_translations_for_potmsgset(potmsgset, potmsgset,
- logger, ztm)
-
- potemplate.source_file_format = TranslationFileFormat.KDEPO
- # Commit a PO template one by one.
- ztm.commit()
-
-
-def migrate_translations_for_kdepo_templates(ztm, logger):
- """Migrate translations for already re-imported KDE PO templates."""
- potemplates = POTemplate.select("""
- source_file_format=%s AND
- POTemplate.id IN
- (SELECT DISTINCT potemplate
- FROM POTMsgSet
- WHERE POTMsgSet.potemplate = POTemplate.id AND
- (POTMsgSet.msgid_plural IS NOT NULL OR
- POTMsgSet.context IS NOT NULL))
- """ % sqlvalues(TranslationFileFormat.KDEPO))
-
- count = potemplates.count()
- index = 0
- for potemplate in potemplates:
- index += 1
- logger.info("Migrating translations for KDE POTemplate %s [%d/%d]" % (
- potemplate.displayname, index, count))
- migrate_kde_potemplate_translations(potemplate, logger, ztm)
-
-
-def migrate_unmigrated_templates_to_kdepo(ztm, logger):
- """Go through all non-KDE PO templates and migrate to KDEPO as needed."""
-
- potemplates = POTemplate.select("""
- source_file_format=%s AND
- POTemplate.id IN
- (SELECT DISTINCT potemplate
- FROM POTMsgSet
- JOIN POMsgID ON POMsgID.id = POTMsgSet.msgid_singular
- WHERE POTMsgSet.potemplate = POTemplate.id AND
- POTMsgSet.msgid_plural IS NULL AND
- (POMsgID.msgid LIKE E'\\\\_n: %%' OR
- POMsgID.msgid LIKE E'\\\\_: %%'))
- """ % sqlvalues(TranslationFileFormat.PO))
-
- count = potemplates.count()
- index = 0
- for potemplate in potemplates:
- index += 1
- logger.info("Migrating POTemplate %s [%d/%d]" % (
- potemplate.displayname, index, count))
- migrate_potemplate(potemplate, logger, ztm)
-
-
-def migrate_potemplates(ztm, logger):
- migrate_translations_for_kdepo_templates(ztm, logger)
- migrate_unmigrated_templates_to_kdepo(ztm, logger)
=== removed file 'lib/lp/translations/scripts/remove_obsolete_translations.py'
--- lib/lp/translations/scripts/remove_obsolete_translations.py 2010-08-20 20:31:18 +0000
+++ lib/lp/translations/scripts/remove_obsolete_translations.py 1970-01-01 00:00:00 +0000
@@ -1,383 +0,0 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-__metaclass__ = type
-
-__all__ = ['RemoveObsoleteTranslations']
-
-from zope.component import getUtility
-from zope.interface import implements
-
-from canonical.database.sqlbase import quote
-from canonical.launchpad.interfaces.launchpad import ILaunchpadCelebrities
-from canonical.launchpad.interfaces.looptuner import ITunableLoop
-from canonical.launchpad.utilities.looptuner import DBLoopTuner
-from canonical.launchpad.webapp.interfaces import (
- IStoreSelector,
- MAIN_STORE,
- MASTER_FLAVOR,
- )
-from lp.registry.interfaces.series import SeriesStatus
-from lp.services.scripts.base import LaunchpadScript
-
-
-collect_order = [
- 'POTemplate',
- 'POFile',
- 'TranslationImportQueueEntry',
- 'POTMsgSet',
- 'TranslationTemplateItem',
- 'TranslationMessage',
- 'POFileTranslator',
- ]
-
-
-remove_order = list(reversed(collect_order))
-
-
-collection_query = """
- DROP TABLE IF EXISTS %(temporary_table)s;
- CREATE TEMP TABLE %(temporary_table)s
- AS SELECT %(table)s.id AS id
- FROM %(table)s
- JOIN %(join_table)s ON %(table)s.%(join_column)s = %(join_table)s.id;
- CREATE UNIQUE INDEX %(temporary_table)s_idx ON %(temporary_table)s (id);
- ANALYZE %(temporary_table)s"""
-
-
-# POTemplate
-collect_obsolete_potemplates_query = """
- DROP TABLE IF EXISTS %(temporary_table)s;
- CREATE TEMP TABLE %(temporary_table)s
- AS SELECT %(table)s.id AS id
- FROM %(table)s
- JOIN %(join_table)s ON %(table)s.%(join_column)s = %(join_table)s.id
- WHERE distroseries.distribution = %(distribution)s
- AND distroseries.releasestatus = %(releasestatus)s;
- CREATE UNIQUE INDEX %(temporary_table)s_idx ON %(temporary_table)s (id);
- ANALYZE %(temporary_table)s"""
-
-
-# Query to remove subset of entries based on a temporary table.
-deletion_query = """
- DELETE FROM %(table)s
- WHERE id IN (
- SELECT id
- FROM %(temporary_table)s
- ORDER BY id
- LIMIT %%d OFFSET %%d
- )"""
-
-
-def commit_transaction(transaction, logger, dry_run=False):
- """Commit ongoing transaction, start a new one.
-
- Pauses process execution to give the database slave a chance
- to keep up.
- """
- if transaction is None:
- return
-
- if not dry_run:
- transaction.commit()
- transaction.begin()
-
-
-class DeletionLoopRunner(object):
- """Generic loop tuner for removal of obsolete translations data."""
- implements(ITunableLoop)
-
- def __init__(self, table_entry, transaction, logger, store,
- dry_run=False):
- """Initialize the loop."""
- size = table_entry['total']
- self.table = table_entry['table']
- self.removal_sql = deletion_query % table_entry
-
- self._txn = transaction
- self._logger = logger
- self._store = store
- self._dry_run = dry_run
- self._iteration_end = size
- self._iterations_done = 0
- self._commit_count = 0
-
- def isDone(self):
- """See `ITunableLoop`."""
- return self._iterations_done >= self._iteration_end
-
- def __call__(self, chunk_size):
- """See `ITunableLoop`."""
- chunk_size = int(chunk_size)
- query = self.removal_sql % (chunk_size, self._iterations_done)
- result = self._store.execute(query)
- self._logger.debug(
- " * Removed another %d %ss (%d of %d)." % (
- result._raw_cursor.rowcount,
- self.table,
- self._iterations_done + result._raw_cursor.rowcount,
- self._iteration_end))
- self._iterations_done += result._raw_cursor.rowcount
- commit_transaction(self._txn, self._logger, dry_run=self._dry_run)
- self._commit_count += 1
-
- def getTotalCommits(self):
- return self._commit_count
-
-
-class TranslationsStatusChecker:
- """Check if translations data is consistent after removal."""
- check_methods = {
- 'productseries_potemplates' : 'collectProductSeriesStats',
- 'other_distroseries_potemplates' : 'collectOtherDistroSeriesStats',
- 'potemplate_potmsgsets' : 'collectPOTemplatePOTMsgSetStats',
- 'potemplate_pofiles' : 'collectPOTemplatePOFileStats',
- 'pofile_pofiletranslators' : 'collectPOFileTranslatorStats',
- 'translationimportqueue_size' : 'getTranslationImportQueueSize',
- }
-
- def __init__(self, store, logger):
- self.store = store
- self.logger = logger
- self.problems = 0
- for attribute in self.check_methods:
- method = getattr(self, self.check_methods[attribute])
- self.logger.debug("Collecting %s..." % attribute)
- setattr(self, attribute, method())
-
- def postCheck(self):
- self.checkObsoletePOTemplates()
-
- for attribute in self.check_methods:
- self.genericCheck(attribute)
-
- if self.problems > 0:
- self.logger.info("%d problems found." % self.problems)
- else:
- self.logger.info("All checks passed.")
-
- def genericCheck(self, attribute_name):
- method = getattr(self, self.check_methods[attribute_name])
- new_stats = method()
- old_stats = getattr(self, attribute_name)
- if old_stats != new_stats:
- if not isinstance(old_stats, int):
- old_stats = len(old_stats)
- new_stats = len(new_stats)
- self.logger.warn(
- "Mismatch in %s (was %d long, now %d)." % (
- attribute_name, old_stats, new_stats))
- self.problems += 1
-
- def checkObsoletePOTemplates(self):
- query = """SELECT COUNT(POTemplate.id) FROM POTemplate
- JOIN DistroSeries
- ON POTemplate.distroseries=DistroSeries.id
- WHERE DistroSeries.releasestatus=%s AND
- DistroSeries.distribution=%s""" % (
- quote(SeriesStatus.OBSOLETE),
- quote(getUtility(ILaunchpadCelebrities).ubuntu))
- result = self.store.execute(query)
- count = result.get_one()
- if int(count[0]) != 0:
- self.logger.warn("\tObsolete POTemplates remaining: %s." % count)
- self.problems += 1
-
- def collectProductSeriesStats(self):
- query = """SELECT ProductSeries.id, COUNT(POTemplate.id)
- FROM ProductSeries
- JOIN POTemplate
- ON POTemplate.productseries=ProductSeries.id
- GROUP BY ProductSeries.id
- ORDER BY ProductSeries.id"""
- result = self.store.execute(query)
- return result.get_all()
-
- def collectOtherDistroSeriesStats(self):
- query = """SELECT DistroSeries.id, COUNT(POTemplate.id)
- FROM DistroSeries
- JOIN POTemplate
- ON POTemplate.distroseries=DistroSeries.id
- WHERE DistroSeries.releasestatus != %s
- GROUP BY DistroSeries.id
- ORDER BY DistroSeries.id""" % quote(
- SeriesStatus.OBSOLETE)
- result = self.store.execute(query)
- return result.get_all()
-
- def collectPOTemplatePOFileStats(self):
- query = """SELECT POTemplate.id, COUNT(POFile.id)
- FROM POTemplate
- LEFT OUTER JOIN DistroSeries
- ON POTemplate.distroseries=DistroSeries.id
- JOIN POFile
- ON POFile.potemplate=POTemplate.id
- WHERE (distroseries IS NOT NULL AND
- DistroSeries.releasestatus != %s) OR
- productseries IS NOT NULL
- GROUP BY POTemplate.id
- ORDER BY POTemplate.id""" % quote(
- SeriesStatus.OBSOLETE)
- result = self.store.execute(query)
- return result.get_all()
-
- def collectPOTemplatePOTMsgSetStats(self):
- query = """SELECT POTemplate.id, COUNT(POTMsgSet.id)
- FROM POTemplate
- LEFT OUTER JOIN DistroSeries
- ON POTemplate.distroseries=DistroSeries.id
- JOIN POTMsgSet
- ON POTMsgSet.potemplate=POTemplate.id
- WHERE (distroseries IS NOT NULL AND
- DistroSeries.releasestatus != %s) OR
- productseries IS NOT NULL
- GROUP BY POTemplate.id
- ORDER BY POTemplate.id""" % quote(
- SeriesStatus.OBSOLETE)
- result = self.store.execute(query)
- return result.get_all()
-
- def collectPOFileTranslatorStats(self):
- query = """SELECT POFile.id, COUNT(POFileTranslator.id)
- FROM POFile
- JOIN POTemplate
- ON POFile.potemplate=POTemplate.id
- LEFT OUTER JOIN DistroSeries
- ON POTemplate.distroseries=DistroSeries.id
- JOIN POFileTranslator
- ON POFileTranslator.pofile=POFile.id
- WHERE (distroseries IS NOT NULL AND
- DistroSeries.releasestatus != %s) OR
- productseries IS NOT NULL
- GROUP BY POFile.id
- ORDER BY POFile.id""" % quote(
- SeriesStatus.OBSOLETE)
- result = self.store.execute(query)
- return result.get_all()
-
- def getTranslationImportQueueSize(self):
- query = """SELECT count(id) FROM TranslationImportQueueEntry"""
- result = self.store.execute(query)
- return int(result.get_one()[0])
-
-
-class RemoveObsoleteTranslations(LaunchpadScript):
-
- def add_my_options(self):
- self.parser.add_option(
- '-d', '--dry-run', action="store_true", dest='dry_run',
- default=False, help="Don't really make any database changes.")
-
- self.parser.add_option(
- '-l', '--loop-timing', dest='loop_time',
- default=5, help="Time in seconds for the loop to run.")
-
- def main(self):
- removal_traits = {
- 'TranslationMessage' :
- { 'table' : 'TranslationMessage',
- 'temporary_table' : 'obsolete_translationmessage',
- 'join_table' : 'obsolete_pofile',
- 'join_column' : 'pofile',
- },
- 'TranslationTemplateItem' :
- { 'table' : 'TranslationTemplateItem',
- 'temporary_table' : 'obsolete_tti',
- 'join_table' : 'obsolete_potemplate',
- 'join_column' : 'potemplate',
- },
- 'POTMsgSet' :
- { 'table' : 'POTMsgSet',
- 'temporary_table' : 'obsolete_potmsgset',
- 'join_table' : 'obsolete_potemplate',
- 'join_column' : 'potemplate',
- },
- 'POFileTranslator' :
- { 'table' : 'POFileTranslator',
- 'temporary_table' : 'obsolete_pofiletranslator',
- 'join_table' : 'obsolete_pofile',
- 'join_column' : 'pofile',
- },
- 'POFile' :
- { 'table' : 'POFile',
- 'temporary_table' : 'obsolete_pofile',
- 'join_table' : 'obsolete_potemplate',
- 'join_column' : 'potemplate',
- },
- 'TranslationImportQueueEntry' :
- { 'table' : 'TranslationImportQueueEntry',
- 'temporary_table' : 'obsolete_queueentries',
- 'join_table' : 'obsolete_potemplate',
- 'join_column' : 'potemplate',
- },
- 'POTemplate' :
- { 'table' : 'POTemplate',
- 'temporary_table' : 'obsolete_potemplate',
- 'join_table' : 'distroseries',
- 'join_column' : 'distroseries',
- 'distribution' :
- quote(getUtility(ILaunchpadCelebrities).ubuntu),
- 'releasestatus' : quote(SeriesStatus.OBSOLETE),
- 'collection_sql' : collect_obsolete_potemplates_query,
- },
- }
-
- if self.options.dry_run:
- self.logger.info("Dry run. Not making any changes.")
-
- self.logger.debug(
- "Removing translations of obsolete Ubuntu versions")
-
- self._commit_count = 0
-
- # Working on the writable master store
- store = getUtility(IStoreSelector).get(MAIN_STORE, MASTER_FLAVOR)
- self._store = store
-
- checker = TranslationsStatusChecker(store, self.logger)
- for table in collect_order:
- entry = removal_traits[table]
- collect_sql = entry.get('collection_sql', collection_query)
- collect = store.execute(collect_sql % entry)
- count = self._count_rows(entry['temporary_table'])
- entry['total'] = count
- self._do_commit()
-
- for table in remove_order:
- entry = removal_traits[table]
- self.logger.info(
- "Removing %d %s rows." % (entry['total'], table))
- loop = DeletionLoopRunner(
- entry, self.txn, self.logger, store,
- dry_run=self.options.dry_run)
- DBLoopTuner(loop, self.options.loop_time, log=self.logger).run()
- self._commit_count += loop.getTotalCommits()
-
- self.logger.info("Done with %d commits." % self._commit_count)
- self.logger.info("Statistics:")
- for table in remove_order:
- self.logger.info("\t%-30s: %d removed" % (
- table, removal_traits[table]['total']))
-
- self.logger.info("Checks:")
- checker.postCheck()
-
- if self.options.dry_run:
- self.txn.abort()
-
- def _count_rows(self, tablename):
- """Helper to count all rows in a table."""
- count_query = "SELECT count(*) FROM %s"
- result = self._store.execute(
- count_query % tablename).get_one()
- return result[0]
-
- def _do_commit(self):
- """Commit ongoing transaction, start a new one.
-
- Pauses process execution to give the database slave a chance
- to keep up.
- """
- commit_transaction(self.txn, self.logger,
- dry_run=self.options.dry_run)
- self._commit_count += 1
=== removed file 'scripts/rosetta/message-sharing-populate-test.py'
--- scripts/rosetta/message-sharing-populate-test.py 2010-07-13 15:40:33 +0000
+++ scripts/rosetta/message-sharing-populate-test.py 1970-01-01 00:00:00 +0000
@@ -1,134 +0,0 @@
-#!/usr/bin/python -S
-#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-# pylint: disable-msg=W0403
-
-"""Test Translation Message Sharing schema initializations.
-
-This script verifies the results of the message-sharing-populate.py
-one-off script. It checks that all schema elements that were added for
-translation message sharing were initialized correctly.
-
-Run this after message-sharing-populate.py, but also occasionally
-afterwards to ensure that the code that initializes the added schema
-elements on new data is working correctly.
-
-The new schema elements are not used yet. Once they are, a next step is
-to start unifying TranslationMessages that are common between
-POTemplates. When that happens, this script will start reporting errors
-at points marked in the source below. Change those points & run again!
-"""
-
-import _pythonpath
-
-from canonical.database.sqlbase import cursor
-from canonical.launchpad.scripts import execute_zcml_for_scripts
-
-
-class SchemaElementsInitializationFailure(Exception):
- """Convenience exception class for errors found in our data."""
- pass
-
-
-def check(query, error_message, expected_result=0):
- """Report error if `query` result does not match `expected_result`."""
- cur = cursor()
- cur.execute(query)
- result, = cur.fetchone()
-
- if result != expected_result:
- counts = "expected %s, found %s" % (expected_result, result)
- raise SchemaElementsInitializationFailure(
- "%s (%s)" % (error_message, counts))
-
-
-def test_schema():
- """Test schema initializations."""
- cur = cursor()
-
- # Are all TranslationMessage.language fields initialized?
- query = """
- SELECT count(*)
- FROM TranslationMessage
- WHERE language IS NULL
- """
- check(query, "Found uninitialized TranslationMessage.language")
-
- # Are all TranslationMessages.potemplate fields initialized?
- # XXX JeroenVermeulen 2008-10-06 spec=message-sharing-migration:
- # potemplate will be allowed to be null later on, when we really
- # start sharing messages. The field means "this message is specific
- # to this potemplate, rather than shared"). Remove this check then.
- query = """
- SELECT count(*)
- FROM TranslationMessage
- WHERE potemplate IS NULL OR language IS NULL
- """
- check(query, "Found uninitialized TranslationMessages.potemplate.")
-
- # Are all TranslationMessages linked to the right languages?
- query = """
- SELECT count(*)
- FROM TranslationMessage
- LEFT JOIN POFile ON POFile.id = TranslationMessage.pofile
- WHERE
- POFile.id IS NULL OR
- POFile.language <> TranslationMessage.language
- """
- check(query, "Found TranslationMessages with incorrect language.")
-
- # Do all POTMsgSets with nonzero sequence numbers have linking-table
- # entries linking them to their POTemplates? (Zero sequence number
- # means "does not participate in this POTemplate," a wart that will
- # go away with this schema change)
- query = """
- SELECT count(*)
- FROM POTMsgSet
- LEFT JOIN TranslationTemplateItem AS i ON i.potmsgset = POTMsgSet.id
- WHERE POTMsgSet.sequence <> 0 AND i.id IS NULL
- """
- check(query, "Found unlinked POTMsgSets.")
-
- # Conversely, is the linking table free of unexpected rows?
- query = """
- SELECT count(*)
- FROM TranslationTemplateItem i
- LEFT JOIN POTMsgSet ON i.potmsgset = POTMsgSet.id
- WHERE POTMsgSet.id IS NULL or POTMsgSet.sequence = 0
- """
- check(query, "Found unexpected TranslationTemplateItem rows.")
-
- # Are all TranslationTemplateItems correct?
- query = """
- SELECT count(*)
- FROM POTMsgSet
- JOIN TranslationTemplateItem AS i ON i.potmsgset = POTMsgSet.id
- WHERE
- i.potemplate <> POTMsgSet.potemplate OR
- i.sequence <> POTMsgSet.sequence
- """
- check(query, "Found incorrect TranslationTemplateItem contents.")
-
- # Is each POTMsgSet linked to no more than one POTemplate?
- # XXX JeroenVermeulen 2008-10-06 spec=message-sharing-migration:
- # Once we start sharing messages across templates, this will become
- # a real m:n relationship between POTMsgSet and POTemplate. At that
- # point, remove his check.
- query = """
- SELECT count(*)
- FROM (
- SELECT count(*)
- FROM TranslationTemplateItem
- GROUP BY potmsgset
- HAVING count(*) > 1
- ) AS shared_potmsgset
- """
- check(query, "Found shared POTMsgSets.")
-
-
-if __name__ == '__main__':
- execute_zcml_for_scripts()
- test_schema()
- print "Done."
=== removed file 'scripts/rosetta/message-sharing-populate.py'
--- scripts/rosetta/message-sharing-populate.py 2010-08-06 07:02:32 +0000
+++ scripts/rosetta/message-sharing-populate.py 1970-01-01 00:00:00 +0000
@@ -1,209 +0,0 @@
-#!/usr/bin/python -S
-#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-# pylint: disable-msg=W0403
-
-"""Populate schema additions for Translations Message Sharing.
-
-This fills two new `TranslationMessage` columns: potemplate, language.
-It also creates linking table entries connecting the existing `POTMsgSet`s
-to their `POTemplate`s.
-
-Since the schema additions are not in use yet, this script doesn't need
-to be careful about grouping by template, preserving any kind of order,
-and so on.
-
-On the other hand, the Python code tree should already be initializing
-the new columns and the linking table by the time this script is run.
-So we do have to be careful not to interfere with that, or stumble when
-some records have already been initialized properly.
-"""
-
-import _pythonpath
-
-from zope.interface import implements
-
-from canonical.database.postgresql import drop_tables
-from canonical.database.sqlbase import cursor, sqlvalues
-from canonical.launchpad.interfaces.looptuner import ITunableLoop
-from lp.services.scripts.base import LaunchpadScript
-from canonical.launchpad.utilities.looptuner import DBLoopTuner
-
-
-class PopulateTranslationMessage:
- """`ITunableLoop` to populate new TranslationMessage columns."""
-
- implements(ITunableLoop)
- def __init__(self, txn, logger):
- self.txn = txn
- self.logger = logger
- self.last_id = 0
-
- cur = cursor()
- cur.execute("""
- SELECT id
- INTO TEMP TABLE temp_todo
- FROM TranslationMessage
- WHERE potemplate IS NULL
- ORDER BY id
- """)
- cur.execute(
- "CREATE UNIQUE INDEX temp_todo__pkey ON temp_todo(id)")
- cur.execute("ANALYZE temp_todo(id)")
-
- cur.execute("SELECT max(id) FROM temp_todo")
- max_id, = cur.fetchone()
- if max_id is None:
- self.finish_id = 0
- else:
- self.finish_id = max_id + 1
-
- def isDone(self):
- """See `ITunableLoop`."""
- done = (self.last_id >= self.finish_id)
- if done:
- drop_tables(cursor(), 'temp_todo')
- return done
-
- def __call__(self, chunk_size):
- """See `ITunableLoop`."""
- chunk_size = int(chunk_size)
-
- cur = cursor()
- cur.execute("""
- SELECT id
- FROM temp_todo
- WHERE id >= %s
- ORDER BY id
- OFFSET %s
- LIMIT 1
- """ % sqlvalues(self.last_id, chunk_size))
- batch_limit = cur.fetchone()
- if batch_limit is None:
- end_id = self.finish_id
- else:
- end_id, = batch_limit
-
- cur.execute("""
- UPDATE TranslationMessage AS Msg
- SET
- potemplate = POFile.potemplate,
- language = POFile.language
- FROM POFile
- WHERE
- POFile.id = Msg.pofile AND
- Msg.potemplate IS NULL AND
- Msg.id IN (
- SELECT id
- FROM temp_todo
- WHERE id >= %s AND id < %s
- )
- """ % sqlvalues(self.last_id, end_id))
- self.logger.info(
- "Updated %d rows: %d - %d." % (
- cur.rowcount, self.last_id, end_id))
- self.txn.commit()
- self.txn.begin()
- self.last_id = end_id
-
-
-class PopulateTranslationTemplateItem:
- """`ITunableLoop` to populate TranslationTemplateItem linking table."""
-
- implements(ITunableLoop)
- def __init__(self, txn, logger):
- self.txn = txn
- self.done = False
- self.logger = logger
- self.last_id = 0
-
- cur = cursor()
- cur.execute("""
- SELECT POTMsgSet.id
- INTO TEMP TABLE temp_todo
- FROM POTMsgSet
- LEFT JOIN TranslationTemplateItem AS ExistingEntry ON
- ExistingEntry.potmsgset = potmsgset.id
- WHERE
- POTMsgSet.sequence > 0 AND
- ExistingEntry.id IS NULL
- ORDER BY id
- """)
- cur.execute(
- "CREATE UNIQUE INDEX temp_todo__pkey ON temp_todo(id)")
- cur.execute("ANALYZE temp_todo(id)")
-
- cur.execute("SELECT max(id) FROM temp_todo")
- max_id, = cur.fetchone()
- if max_id is None:
- self.finish_id = 0
- else:
- self.finish_id = max_id + 1
-
- def isDone(self):
- """See `ITunableLoop`."""
- done = (self.last_id >= self.finish_id)
- if done:
- drop_tables(cursor(), 'temp_todo')
- return done
-
- def __call__(self, chunk_size):
- """See `ITunableLoop`."""
- chunk_size = int(chunk_size)
- cur = cursor()
-
- cur.execute("""
- SELECT id
- FROM temp_todo
- WHERE id >= %s
- ORDER BY id
- OFFSET %s
- LIMIT 1
- """ % sqlvalues(self.last_id, chunk_size))
- batch_limit = cur.fetchone()
- if batch_limit is None:
- end_id = self.finish_id
- else:
- end_id, = batch_limit
-
- cur.execute("""
- INSERT INTO TranslationTemplateItem(
- potemplate, sequence, potmsgset)
- SELECT POTMsgSet.potemplate, POTMsgSet.sequence, POTMsgSet.id
- FROM POTMsgSet
- LEFT JOIN TranslationTemplateItem AS ExistingEntry ON
- ExistingEntry.potmsgset = potmsgset.id
- WHERE
- POTMsgSet.id >= %s AND
- POTMsgSet.id < %s AND
- POTMsgSet.sequence > 0 AND
- ExistingEntry.id IS NULL
- """ % sqlvalues(self.last_id, end_id))
- self.logger.info("Inserted %d rows." % cur.rowcount)
- self.txn.commit()
- self.txn.begin()
- self.last_id = end_id
-
-
-class PopulateMessageSharingSchema(LaunchpadScript):
- description = (
- "Populate columns and linking table added for Translations Message "
- "sharing.")
-
- def main(self):
- self.logger.info("Populating new TranslationMessage columns.")
- tm_loop = PopulateTranslationMessage(self.txn, self.logger)
- DBLoopTuner(tm_loop, 2, log=self.logger).run()
-
- self.logger.info("Populating TranslationTemplateItem.")
- tti_loop = PopulateTranslationTemplateItem(self.txn, self.logger)
- DBLoopTuner(tti_loop, 2, log=self.logger).run()
-
-
-if __name__ == '__main__':
- script = PopulateMessageSharingSchema(
- 'canonical.launchpad.scripts.message-sharing-populate')
- script.run()
-
=== removed file 'scripts/rosetta/remove-obsolete-translations.py'
--- scripts/rosetta/remove-obsolete-translations.py 2010-04-27 19:48:39 +0000
+++ scripts/rosetta/remove-obsolete-translations.py 1970-01-01 00:00:00 +0000
@@ -1,21 +0,0 @@
-#!/usr/bin/python -S
-#
-# Copyright 2009 Canonical Ltd. This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-# pylint: disable-msg=W0403
-# (Suppressing pylint "relative import" warning 0403 for _pythonpath)
-
-__metaclass__ = type
-
-import _pythonpath
-
-from lp.translations.scripts.remove_obsolete_translations import (
- RemoveObsoleteTranslations)
-
-
-if __name__ == '__main__':
- script = RemoveObsoleteTranslations(
- 'canonical.launchpad.scripts.remove-obsolete-translations',
- dbuser='rosettaadmin')
- script.run()
=== modified file 'utilities/migrater/file-ownership.txt'
--- utilities/migrater/file-ownership.txt 2010-10-31 20:18:45 +0000
+++ utilities/migrater/file-ownership.txt 2010-11-16 13:00:24 +0000
@@ -604,7 +604,6 @@
reg ./scripts/listteammembers.py
./scripts/logger.py
./scripts/loghandlers.py
- ./scripts/migrate_kde_potemplates.py
./scripts/mlistimport.py
./scripts/oops.py
./scripts/packagecopier.py
@@ -623,7 +622,6 @@
./scripts/publishdistro.py
./scripts/questionexpiration.py
./scripts/queue.py
- ./scripts/remove_obsolete_translations.py
./scripts/remove_translations.py
./scripts/revisionkarma.py
./scripts/runlaunchpad.py
