launchpad-reviewers team mailing list archive

[Colin Watson <cjwatson@xxxxxxxxxxxxx>]

Colin Watson has proposed merging lp:~cjwatson/launchpad/xz-indexes into lp:launchpad with lp:~cjwatson/launchpad/distroseries-publishing-options-2 as a prerequisite.

Commit message:
Make index compression types configurable per-series, and add xz support.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)
Related bugs:
  Bug #1517510 in Launchpad itself: "Add xz-compressed archive indexes"
  https://bugs.launchpad.net/launchpad/+bug/1517510

For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/xz-indexes/+merge/285238

Make index compression types configurable per-series, and add xz support.

I have amd64 and i386 eggs of backports.lzma which I can check into lp-source-dependencies before landing this.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/xz-indexes into lp:launchpad.

=== modified file 'lib/lp/archivepublisher/model/ftparchive.py'
--- lib/lp/archivepublisher/model/ftparchive.py	2016-02-04 19:46:52 +0000
+++ lib/lp/archivepublisher/model/ftparchive.py	2016-02-05 20:34:36 +0000
@@ -27,6 +27,7 @@
 from lp.services.osutils import write_file
 from lp.soyuz.enums import (
     BinaryPackageFormat,
+    IndexCompressionType,
     PackagePublishingStatus,
     )
 from lp.soyuz.model.binarypackagebuild import BinaryPackageBuild
@@ -67,10 +68,7 @@
 
 Default
 {
-    Packages::Compress "gzip bzip2";
-    Sources::Compress "gzip bzip2";
     Contents::Compress "gzip";
-    Translation::Compress "gzip bzip2";
     DeLinkLimit 0;
     MaxContentsChange 12000;
     FileMode 0644;
@@ -95,6 +93,9 @@
     SrcOverride "override.%(DISTRORELEASE)s.$(SECTION).src";
     %(HIDEEXTRA)sExtraOverride "override.%(DISTRORELEASE)s.extra.$(SECTION)";
     Packages::Extensions "%(EXTENSIONS)s";
+    Packages::Compress "%(COMPRESSORS)s";
+    Sources::Compress "%(COMPRESSORS)s";
+    Translation::Compress "%(COMPRESSORS)s";
     BinCacheDB "packages%(CACHEINSERT)s-$(ARCH).db";
     SrcCacheDB "sources%(CACHEINSERT)s.db";
     Contents " ";
@@ -115,6 +116,13 @@
 
 CLEANUP_FREQUENCY = 60 * 60 * 24
 
+COMPRESSOR_TO_CONFIG = {
+    IndexCompressionType.UNCOMPRESSED: '.',
+    IndexCompressionType.GZIP: 'gzip',
+    IndexCompressionType.BZIP2: 'bzip2',
+    IndexCompressionType.XZ: 'xz',
+    }
+
 
 class AptFTPArchiveFailure(Exception):
     """Failure while running apt-ftparchive."""
@@ -743,7 +751,8 @@
 
         self.writeAptConfig(
             apt_config, suite, comps, archs,
-            distroseries.include_long_descriptions)
+            distroseries.include_long_descriptions,
+            distroseries.index_compressors)
 
         # XXX: 2006-08-24 kiko: Why do we do this directory creation here?
         for comp in comps:
@@ -759,8 +768,10 @@
                         component_path, subcomp, "binary-" + arch))
 
     def writeAptConfig(self, apt_config, suite, comps, archs,
-                       include_long_descriptions):
+                       include_long_descriptions, index_compressors):
         self.log.debug("Generating apt config for %s" % suite)
+        compressors = " ".join(
+            COMPRESSOR_TO_CONFIG[c] for c in index_compressors)
         apt_config.write(STANZA_TEMPLATE % {
                          "LISTPATH": self._config.overrideroot,
                          "DISTRORELEASE": suite,
@@ -769,6 +780,7 @@
                          "ARCHITECTURES": " ".join(archs + ["source"]),
                          "SECTIONS": " ".join(comps),
                          "EXTENSIONS": ".deb",
+                         "COMPRESSORS": compressors,
                          "CACHEINSERT": "",
                          "DISTS": os.path.basename(self._config.distsroot),
                          "HIDEEXTRA": "",
@@ -787,6 +799,7 @@
                         "ARCHITECTURES": " ".join(archs),
                         "SECTIONS": subcomp,
                         "EXTENSIONS": '.%s' % SUBCOMPONENT_TO_EXT[subcomp],
+                        "COMPRESSORS": compressors,
                         "CACHEINSERT": "-%s" % subcomp,
                         "DISTS": os.path.basename(self._config.distsroot),
                         "HIDEEXTRA": "// ",
@@ -828,7 +841,7 @@
                 comps.add(comp.name)
         self.writeAptConfig(
             apt_config, "nonexistent-suite", sorted(comps), sorted(archs),
-            True)
+            True, [IndexCompressionType.UNCOMPRESSED])
 
         with open(apt_config_filename, "w") as fp:
             fp.write(apt_config.getvalue())

=== modified file 'lib/lp/archivepublisher/publishing.py'
--- lib/lp/archivepublisher/publishing.py	2016-02-05 02:12:06 +0000
+++ lib/lp/archivepublisher/publishing.py	2016-02-05 20:34:36 +0000
@@ -118,13 +118,15 @@
         return path[:-len('.gz')]
     elif path.endswith('.bz2'):
         return path[:-len('.bz2')]
+    elif path.endswith('.xz'):
+        return path[:-len('.xz')]
     else:
         return path
 
 
 def get_suffixed_indices(path):
     """Return a set of paths to compressed copies of the given index."""
-    return set([path + suffix for suffix in ('', '.gz', '.bz2')])
+    return set([path + suffix for suffix in ('', '.gz', '.bz2', '.xz')])
 
 
 def _getDiskPool(pubconf, log):
@@ -685,11 +687,11 @@
             translation_en = RepositoryIndexFile(
                 os.path.join(self._config.distsroot, suite_name,
                              component.name, "i18n", "Translation-en"),
-                self._config.temproot)
+                self._config.temproot, distroseries.index_compressors)
 
         source_index = RepositoryIndexFile(
             get_sources_path(self._config, suite_name, component),
-            self._config.temproot)
+            self._config.temproot, distroseries.index_compressors)
 
         for spp in distroseries.getSourcePackagePublishing(
                 pocket, component, self.archive):
@@ -710,13 +712,13 @@
             indices = {}
             indices[None] = RepositoryIndexFile(
                 get_packages_path(self._config, suite_name, component, arch),
-                self._config.temproot)
+                self._config.temproot, distroseries.index_compressors)
 
             for subcomp in self.subcomponents:
                 indices[subcomp] = RepositoryIndexFile(
                     get_packages_path(
                         self._config, suite_name, component, arch, subcomp),
-                    self._config.temproot)
+                    self._config.temproot, distroseries.index_compressors)
 
             for bpp in distroseries.getBinaryPackagePublishing(
                     arch.architecturetag, pocket, component, self.archive):

=== modified file 'lib/lp/archivepublisher/tests/apt-data/apt.conf'
--- lib/lp/archivepublisher/tests/apt-data/apt.conf	2016-02-04 19:46:52 +0000
+++ lib/lp/archivepublisher/tests/apt-data/apt.conf	2016-02-05 20:34:36 +0000
@@ -8,10 +8,7 @@
 
 Default
 {
-    Packages::Compress "gzip bzip2";
-    Sources::Compress "gzip bzip2";
     Contents::Compress "gzip";
-    Translation::Compress "gzip bzip2";
     DeLinkLimit 0;
     MaxContentsChange 12000;
     FileMode 0644;
@@ -34,6 +31,9 @@
     SrcOverride "override.hoary-test.$(SECTION).src";
     ExtraOverride "override.hoary-test.extra.$(SECTION)";
     Packages::Extensions ".deb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-$(ARCH).db";
     SrcCacheDB "sources.db";
     Contents " ";
@@ -51,6 +51,9 @@
     SrcOverride "override.hoary-test.main.$(SECTION).src";
     // ExtraOverride "override.hoary-test.main.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -68,6 +71,9 @@
     SrcOverride "override.hoary-test.restricted.$(SECTION).src";
     // ExtraOverride "override.hoary-test.restricted.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -85,6 +91,9 @@
     SrcOverride "override.hoary-test.universe.$(SECTION).src";
     // ExtraOverride "override.hoary-test.universe.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -102,6 +111,9 @@
     SrcOverride "override.hoary-test.multiverse.$(SECTION).src";
     // ExtraOverride "override.hoary-test.multiverse.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -119,6 +131,9 @@
     SrcOverride "override.hoary-test-security.$(SECTION).src";
     ExtraOverride "override.hoary-test-security.extra.$(SECTION)";
     Packages::Extensions ".deb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-$(ARCH).db";
     SrcCacheDB "sources.db";
     Contents " ";
@@ -136,6 +151,9 @@
     SrcOverride "override.hoary-test-security.main.$(SECTION).src";
     // ExtraOverride "override.hoary-test-security.main.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -153,6 +171,9 @@
     SrcOverride "override.hoary-test-security.restricted.$(SECTION).src";
     // ExtraOverride "override.hoary-test-security.restricted.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -170,6 +191,9 @@
     SrcOverride "override.hoary-test-security.universe.$(SECTION).src";
     // ExtraOverride "override.hoary-test-security.universe.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -187,6 +211,9 @@
     SrcOverride "override.hoary-test-security.multiverse.$(SECTION).src";
     // ExtraOverride "override.hoary-test-security.multiverse.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -204,6 +231,9 @@
     SrcOverride "override.hoary-test-updates.$(SECTION).src";
     ExtraOverride "override.hoary-test-updates.extra.$(SECTION)";
     Packages::Extensions ".deb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-$(ARCH).db";
     SrcCacheDB "sources.db";
     Contents " ";
@@ -221,6 +251,9 @@
     SrcOverride "override.hoary-test-updates.main.$(SECTION).src";
     // ExtraOverride "override.hoary-test-updates.main.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -238,6 +271,9 @@
     SrcOverride "override.hoary-test-updates.restricted.$(SECTION).src";
     // ExtraOverride "override.hoary-test-updates.restricted.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -255,6 +291,9 @@
     SrcOverride "override.hoary-test-updates.universe.$(SECTION).src";
     // ExtraOverride "override.hoary-test-updates.universe.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -272,6 +311,9 @@
     SrcOverride "override.hoary-test-updates.multiverse.$(SECTION).src";
     // ExtraOverride "override.hoary-test-updates.multiverse.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -289,6 +331,9 @@
     SrcOverride "override.hoary-test-proposed.$(SECTION).src";
     ExtraOverride "override.hoary-test-proposed.extra.$(SECTION)";
     Packages::Extensions ".deb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-$(ARCH).db";
     SrcCacheDB "sources.db";
     Contents " ";
@@ -306,6 +351,9 @@
     SrcOverride "override.hoary-test-proposed.main.$(SECTION).src";
     // ExtraOverride "override.hoary-test-proposed.main.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -323,6 +371,9 @@
     SrcOverride "override.hoary-test-proposed.restricted.$(SECTION).src";
     // ExtraOverride "override.hoary-test-proposed.restricted.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -340,6 +391,9 @@
     SrcOverride "override.hoary-test-proposed.universe.$(SECTION).src";
     // ExtraOverride "override.hoary-test-proposed.universe.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -357,6 +411,9 @@
     SrcOverride "override.hoary-test-proposed.multiverse.$(SECTION).src";
     // ExtraOverride "override.hoary-test-proposed.multiverse.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -374,6 +431,9 @@
     SrcOverride "override.hoary-test-backports.$(SECTION).src";
     ExtraOverride "override.hoary-test-backports.extra.$(SECTION)";
     Packages::Extensions ".deb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-$(ARCH).db";
     SrcCacheDB "sources.db";
     Contents " ";
@@ -391,6 +451,9 @@
     SrcOverride "override.hoary-test-backports.main.$(SECTION).src";
     // ExtraOverride "override.hoary-test-backports.main.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -408,6 +471,9 @@
     SrcOverride "override.hoary-test-backports.restricted.$(SECTION).src";
     // ExtraOverride "override.hoary-test-backports.restricted.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -425,6 +491,9 @@
     SrcOverride "override.hoary-test-backports.universe.$(SECTION).src";
     // ExtraOverride "override.hoary-test-backports.universe.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -442,6 +511,9 @@
     SrcOverride "override.hoary-test-backports.multiverse.$(SECTION).src";
     // ExtraOverride "override.hoary-test-backports.multiverse.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -459,6 +531,9 @@
     SrcOverride "override.breezy-autotest.$(SECTION).src";
     ExtraOverride "override.breezy-autotest.extra.$(SECTION)";
     Packages::Extensions ".deb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-$(ARCH).db";
     SrcCacheDB "sources.db";
     Contents " ";
@@ -476,6 +551,9 @@
     SrcOverride "override.breezy-autotest-security.$(SECTION).src";
     ExtraOverride "override.breezy-autotest-security.extra.$(SECTION)";
     Packages::Extensions ".deb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-$(ARCH).db";
     SrcCacheDB "sources.db";
     Contents " ";
@@ -493,6 +571,9 @@
     SrcOverride "override.breezy-autotest-updates.$(SECTION).src";
     ExtraOverride "override.breezy-autotest-updates.extra.$(SECTION)";
     Packages::Extensions ".deb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-$(ARCH).db";
     SrcCacheDB "sources.db";
     Contents " ";
@@ -510,6 +591,9 @@
     SrcOverride "override.breezy-autotest-proposed.$(SECTION).src";
     ExtraOverride "override.breezy-autotest-proposed.extra.$(SECTION)";
     Packages::Extensions ".deb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-$(ARCH).db";
     SrcCacheDB "sources.db";
     Contents " ";
@@ -527,6 +611,9 @@
     SrcOverride "override.breezy-autotest-backports.$(SECTION).src";
     ExtraOverride "override.breezy-autotest-backports.extra.$(SECTION)";
     Packages::Extensions ".deb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-$(ARCH).db";
     SrcCacheDB "sources.db";
     Contents " ";

=== modified file 'lib/lp/archivepublisher/tests/apt-data/apt_conf_single_empty_suite_test'
--- lib/lp/archivepublisher/tests/apt-data/apt_conf_single_empty_suite_test	2016-02-04 19:46:52 +0000
+++ lib/lp/archivepublisher/tests/apt-data/apt_conf_single_empty_suite_test	2016-02-05 20:34:36 +0000
@@ -8,10 +8,7 @@
 
 Default
 {
-    Packages::Compress "gzip bzip2";
-    Sources::Compress "gzip bzip2";
     Contents::Compress "gzip";
-    Translation::Compress "gzip bzip2";
     DeLinkLimit 0;
     MaxContentsChange 12000;
     FileMode 0644;
@@ -34,6 +31,9 @@
     SrcOverride "override.hoary-test-updates.$(SECTION).src";
     ExtraOverride "override.hoary-test-updates.extra.$(SECTION)";
     Packages::Extensions ".deb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-$(ARCH).db";
     SrcCacheDB "sources.db";
     Contents " ";
@@ -51,6 +51,9 @@
     SrcOverride "override.hoary-test-updates.main.$(SECTION).src";
     // ExtraOverride "override.hoary-test-updates.main.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -68,6 +71,9 @@
     SrcOverride "override.hoary-test-updates.restricted.$(SECTION).src";
     // ExtraOverride "override.hoary-test-updates.restricted.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -85,6 +91,9 @@
     SrcOverride "override.hoary-test-updates.universe.$(SECTION).src";
     // ExtraOverride "override.hoary-test-updates.universe.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";
@@ -102,6 +111,9 @@
     SrcOverride "override.hoary-test-updates.multiverse.$(SECTION).src";
     // ExtraOverride "override.hoary-test-updates.multiverse.extra.$(SECTION)";
     Packages::Extensions ".udeb";
+    Packages::Compress "gzip bzip2";
+    Sources::Compress "gzip bzip2";
+    Translation::Compress "gzip bzip2";
     BinCacheDB "packages-debian-installer-$(ARCH).db";
     SrcCacheDB "sources-debian-installer.db";
     Contents " ";

=== modified file 'lib/lp/archivepublisher/tests/test_ftparchive.py'
--- lib/lp/archivepublisher/tests/test_ftparchive.py	2016-02-04 19:46:52 +0000
+++ lib/lp/archivepublisher/tests/test_ftparchive.py	2016-02-05 20:34:36 +0000
@@ -459,7 +459,7 @@
         # Test that a publisher run now will generate an empty apt
         # config and nothing else.
         apt_conf = fa.generateConfig()
-        assert len(file(apt_conf).readlines()) == 25
+        self.assertEqual(22, len(file(apt_conf).readlines()))
 
         # XXX cprov 2007-03-21: see above, do not run a-f on dev machines.
         fa.runApt(apt_conf)

=== modified file 'lib/lp/archivepublisher/tests/test_publisher.py'
--- lib/lp/archivepublisher/tests/test_publisher.py	2016-02-04 19:46:52 +0000
+++ lib/lp/archivepublisher/tests/test_publisher.py	2016-02-05 20:34:36 +0000
@@ -18,6 +18,10 @@
 import time
 
 from debian.deb822 import Release
+try:
+    import lzma
+except ImportError:
+    from backports import lzma
 from testtools.matchers import ContainsAll
 import transaction
 from zope.component import getUtility
@@ -56,6 +60,7 @@
     ArchivePurpose,
     ArchiveStatus,
     BinaryPackageFormat,
+    IndexCompressionType,
     PackagePublishingStatus,
     PackageUploadStatus,
     )
@@ -1014,11 +1019,10 @@
         """Assert that the various compressed versions of a file are equal.
 
         Check that the various versions of a compressed file, such as
-        Packages.gz/Packages.bz2 and Sources.gz/Sources.bz2, and bz2
-        variations, all have identical contents.  The file paths are
-        relative to breezy-autotest/main under the archive_publisher's
-        configured dist root.  'breezy-autotest' is our test distroseries
-        name.
+        Packages.{gz,bz2,xz} and Sources.{gz,bz2,xz} all have identical
+        contents.  The file paths are relative to breezy-autotest/main under
+        the archive_publisher's configured dist root.  'breezy-autotest' is
+        our test distroseries name.
 
         The contents of the uncompressed file is returned as a list of lines
         in the file.
@@ -1033,6 +1037,8 @@
                 open_func = gzip.open
             elif suffix == '.bz2':
                 open_func = bz2.BZ2File
+            elif suffix == '.xz':
+                open_func = lzma.LZMAFile
             else:
                 open_func = open
             with open_func(index_base_path + suffix) as index_file:
@@ -1044,7 +1050,7 @@
         return all_contents[0]
 
     def setupPPAArchiveIndexTest(self, long_descriptions=True,
-                                 feature_flag=False):
+                                 feature_flag=False, index_compressors=None):
         # Setup for testPPAArchiveIndex tests
         allowed_suites = []
 
@@ -1082,10 +1088,12 @@
             self.assertEqual('enabled', getFeatureFlag(
                 'soyuz.ppa.separate_long_descriptions'))
 
+        ds = self.ubuntutest.getSeries('breezy-autotest')
         if not long_descriptions:
             # Make sure that NMAF generates i18n/Translation-en* files.
-            ds = self.ubuntutest.getSeries('breezy-autotest')
             ds.include_long_descriptions = False
+        if index_compressors is not None:
+            ds.index_compressors = index_compressors
 
         archive_publisher.A_publish(False)
         self.layer.txn.commit()
@@ -1218,6 +1226,8 @@
             os.path.join(i18n_path, 'Translation-en.gz')))
         self.assertFalse(os.path.exists(
             os.path.join(i18n_path, 'Translation-en.bz2')))
+        self.assertFalse(os.path.exists(
+            os.path.join(i18n_path, 'Translation-en.xz')))
 
         # remove PPA root
         shutil.rmtree(config.personalpackagearchive.root)
@@ -1237,6 +1247,8 @@
             os.path.join(i18n_path, 'Translation-en.gz')))
         self.assertFalse(os.path.exists(
             os.path.join(i18n_path, 'Translation-en.bz2')))
+        self.assertFalse(os.path.exists(
+            os.path.join(i18n_path, 'Translation-en.xz')))
 
         # remove PPA root
         shutil.rmtree(config.personalpackagearchive.root)
@@ -1401,10 +1413,32 @@
         with open(release_path) as release_file:
             content = release_file.read()
             self.assertIn('main/i18n/Translation-en.bz2', content)
+            self.assertIn('main/i18n/Translation-en.gz', content)
 
         # remove PPA root
         shutil.rmtree(config.personalpackagearchive.root)
 
+    def testPPAArchiveIndexCompressors(self):
+        # Archive index generation honours DistroSeries.index_compressors.
+        archive_publisher = self.setupPPAArchiveIndexTest(
+            long_descriptions=False, feature_flag=True,
+            index_compressors=[
+                IndexCompressionType.UNCOMPRESSED, IndexCompressionType.XZ])
+        suite_path = os.path.join(
+            archive_publisher._config.distsroot, 'breezy-autotest', 'main')
+        for uncompressed_file_path in (
+                os.path.join('source', 'Sources'),
+                os.path.join('binary-i386', 'Packages'),
+                os.path.join('debian-installer', 'binary-i386', 'Packages'),
+                os.path.join('debug', 'binary-i386', 'Packages'),
+                os.path.join('i18n', 'Translation-en'),
+                ):
+            for suffix in ('bz2', 'gz'):
+                self.assertFalse(os.path.exists(os.path.join(
+                    suite_path, '%s.%s' % (uncompressed_file_path, suffix))))
+            self._checkCompressedFiles(
+                archive_publisher, uncompressed_file_path, ['.xz'])
+
     def checkDirtyPockets(self, publisher, expected):
         """Check dirty_pockets contents of a given publisher."""
         sorted_dirty_pockets = sorted(list(publisher.dirty_pockets))
@@ -1768,6 +1802,7 @@
             content = release_file.read()
             for component in components:
                 self.assertIn(component + '/i18n/Translation-en.bz2', content)
+                self.assertIn(component + '/i18n/Translation-en.gz', content)
 
     def testReleaseFileForContents(self):
         """Test Release file writing for Contents files."""
@@ -1942,7 +1977,8 @@
 
         # Write compressed versions of a zero-length Translation-en file.
         translation_en_index = RepositoryIndexFile(
-            os.path.join(i18n_root, 'Translation-en'), self.config.temproot)
+            os.path.join(i18n_root, 'Translation-en'), self.config.temproot,
+            self.ubuntutest['breezy-autotest'].index_compressors)
         translation_en_index.close()
 
         all_files = set()

=== modified file 'lib/lp/archivepublisher/tests/test_repositoryindexfile.py'
--- lib/lp/archivepublisher/tests/test_repositoryindexfile.py	2016-02-04 19:46:52 +0000
+++ lib/lp/archivepublisher/tests/test_repositoryindexfile.py	2016-02-05 20:34:36 +0000
@@ -13,7 +13,13 @@
 import tempfile
 import unittest
 
+try:
+    import lzma
+except ImportError:
+    from backports import lzma
+
 from lp.archivepublisher.utils import RepositoryIndexFile
+from lp.soyuz.enums import IndexCompressionType
 
 
 class TestRepositoryArchiveIndex(unittest.TestCase):
@@ -32,14 +38,20 @@
         for path in [self.root, self.temp_root]:
             shutil.rmtree(path)
 
-    def getRepoFile(self, filename):
+    def getRepoFile(self, filename, compressors=None):
         """Return a `RepositoryIndexFile` for the given filename.
 
         The `RepositoryIndexFile` is created with the test 'root' and
         'temp_root'.
         """
+        if compressors is None:
+            compressors = [
+                IndexCompressionType.GZIP,
+                IndexCompressionType.BZIP2,
+                IndexCompressionType.XZ,
+                ]
         return RepositoryIndexFile(
-            os.path.join(self.root, filename), self.temp_root)
+            os.path.join(self.root, filename), self.temp_root, compressors)
 
     def testWorkflow(self):
         """`RepositoryIndexFile` workflow.
@@ -58,15 +70,16 @@
         repo_file = self.getRepoFile('boing')
 
         self.assertEqual(0, len(os.listdir(self.root)))
-        self.assertEqual(2, len(os.listdir(self.temp_root)))
+        self.assertEqual(3, len(os.listdir(self.temp_root)))
 
         repo_file.close()
 
-        self.assertEqual(2, len(os.listdir(self.root)))
+        self.assertEqual(3, len(os.listdir(self.root)))
         self.assertEqual(0, len(os.listdir(self.temp_root)))
 
         resulting_files = sorted(os.listdir(self.root))
-        self.assertEqual(['boing.bz2', 'boing.gz'], resulting_files)
+        self.assertEqual(
+            ['boing.bz2', 'boing.gz', 'boing.xz'], resulting_files)
 
         for filename in resulting_files:
             file_path = os.path.join(self.root, filename)
@@ -89,10 +102,21 @@
         gzip_content = gzip.open(os.path.join(self.root, 'boing.gz')).read()
         bz2_content = bz2.decompress(
             open(os.path.join(self.root, 'boing.bz2')).read())
+        xz_content = lzma.open(os.path.join(self.root, 'boing.xz')).read()
 
         self.assertEqual(gzip_content, bz2_content)
+        self.assertEqual(gzip_content, xz_content)
         self.assertEqual('hello', gzip_content)
 
+    def testCompressors(self):
+        """`RepositoryIndexFile` honours the supplied list of compressors."""
+        repo_file = self.getRepoFile(
+            'boing',
+            compressors=[
+                IndexCompressionType.UNCOMPRESSED, IndexCompressionType.XZ])
+        repo_file.close()
+        self.assertEqual(['boing', 'boing.xz'], sorted(os.listdir(self.root)))
+
     def testUnreferencing(self):
         """`RepositoryIndexFile` unreferencing.
 
@@ -102,7 +126,7 @@
         repo_file = self.getRepoFile('boing')
 
         self.assertEqual(0, len(os.listdir(self.root)))
-        self.assertEqual(2, len(os.listdir(self.temp_root)))
+        self.assertEqual(3, len(os.listdir(self.temp_root)))
 
         del repo_file
 
@@ -112,15 +136,20 @@
     def testRootCreation(self):
         """`RepositoryIndexFile` creates given 'root' path if necessary."""
         missing_root = os.path.join(self.root, 'donotexist')
+        compressors = [
+            IndexCompressionType.GZIP,
+            IndexCompressionType.BZIP2,
+            IndexCompressionType.XZ,
+            ]
         repo_file = RepositoryIndexFile(
-            os.path.join(missing_root, 'boing'), self.temp_root)
+            os.path.join(missing_root, 'boing'), self.temp_root, compressors)
 
         self.assertFalse(os.path.exists(missing_root))
 
         repo_file.close()
 
         self.assertEqual(
-            ['boing.bz2', 'boing.gz'],
+            ['boing.bz2', 'boing.gz', 'boing.xz'],
             sorted(os.listdir(missing_root)))
 
     def testMissingTempRoot(self):
@@ -128,7 +157,8 @@
         missing_temp_root = os.path.join(self.temp_root, 'donotexist')
         self.assertRaises(
             AssertionError, RepositoryIndexFile,
-            os.path.join(self.root, 'boing'), missing_temp_root)
+            os.path.join(self.root, 'boing'), missing_temp_root,
+            [IndexCompressionType.UNCOMPRESSED])
 
     def testRemoveOld(self):
         """`RepositoryIndexFile` removes old index files."""
@@ -139,4 +169,5 @@
         repo_file = self.getRepoFile('boing')
         repo_file.close()
         self.assertEqual(
-            ['boing.bz2', 'boing.gz'], sorted(os.listdir(self.root)))
+            ['boing.bz2', 'boing.gz', 'boing.xz'],
+            sorted(os.listdir(self.root)))

=== modified file 'lib/lp/archivepublisher/utils.py'
--- lib/lp/archivepublisher/utils.py	2016-02-04 19:46:52 +0000
+++ lib/lp/archivepublisher/utils.py	2016-02-05 20:34:36 +0000
@@ -17,7 +17,15 @@
 import stat
 import tempfile
 
-from lp.soyuz.enums import ArchivePurpose
+try:
+    import lzma
+except ImportError:
+    from backports import lzma
+
+from lp.soyuz.enums import (
+    ArchivePurpose,
+    IndexCompressionType,
+    )
 from lp.soyuz.interfaces.archive import default_name_by_purpose
 
 
@@ -38,6 +46,8 @@
 
 class PlainTempFile:
 
+    # Enumerated identifier.
+    compression_type = IndexCompressionType.UNCOMPRESSED
     # Filename suffix.
     suffix = ''
     # File path built on initialization.
@@ -71,6 +81,7 @@
 
 
 class GzipTempFile(PlainTempFile):
+    compression_type = IndexCompressionType.GZIP
     suffix = '.gz'
 
     def _buildFile(self, fd):
@@ -78,6 +89,7 @@
 
 
 class Bzip2TempFile(PlainTempFile):
+    compression_type = IndexCompressionType.BZIP2
     suffix = '.bz2'
 
     def _buildFile(self, fd):
@@ -85,14 +97,23 @@
         return bz2.BZ2File(self.path, mode='wb')
 
 
+class XZTempFile(PlainTempFile):
+    compression_type = IndexCompressionType.XZ
+    suffix = '.xz'
+
+    def _buildFile(self, fd):
+        os.close(fd)
+        return lzma.LZMAFile(self.path, mode='wb', format=lzma.FORMAT_XZ)
+
+
 class RepositoryIndexFile:
     """Facilitates the publication of repository index files.
 
     It allows callsites to publish index files in different medias
-    (plain, gzip and bzip2) transparently and atomically.
+    (plain, gzip, bzip2, and xz) transparently and atomically.
     """
 
-    def __init__(self, path, temp_root):
+    def __init__(self, path, temp_root, compressors):
         """Store repositories destinations and filename.
 
         The given 'temp_root' needs to exist; on the other hand, the
@@ -105,13 +126,14 @@
         self.root, filename = os.path.split(path)
         assert os.path.exists(temp_root), 'Temporary root does not exist.'
 
-        self.index_files = (
-            GzipTempFile(temp_root, filename),
-            Bzip2TempFile(temp_root, filename),
-            )
-        self.old_index_files = (
-            PlainTempFile(temp_root, filename, auto_open=False),
-            )
+        self.index_files = []
+        self.old_index_files = []
+        for cls in (PlainTempFile, GzipTempFile, Bzip2TempFile, XZTempFile):
+            if cls.compression_type in compressors:
+                self.index_files.append(cls(temp_root, filename))
+            else:
+                self.old_index_files.append(
+                    cls(temp_root, filename, auto_open=False))
 
     def write(self, content):
         """Write contents to all target medias."""

=== modified file 'lib/lp/registry/configure.zcml'
--- lib/lp/registry/configure.zcml	2016-01-26 15:47:37 +0000
+++ lib/lp/registry/configure.zcml	2016-02-05 20:34:36 +0000
@@ -1,4 +1,4 @@
-<!-- Copyright 2009-2015 Canonical Ltd.  This software is licensed under the
+<!-- Copyright 2009-2016 Canonical Ltd.  This software is licensed under the
      GNU Affero General Public License version 3 (see the file LICENSE).
 -->
 
@@ -317,6 +317,7 @@
                 driver
                 backports_not_automatic
                 include_long_descriptions
+                index_compressors
                 inherit_overrides_from_parents"/>
 
         <!-- NB: check with SABDFL before modifying these, there is potential to

=== modified file 'lib/lp/registry/interfaces/distroseries.py'
--- lib/lp/registry/interfaces/distroseries.py	2015-10-13 13:22:08 +0000
+++ lib/lp/registry/interfaces/distroseries.py	2016-02-05 20:34:36 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2015 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2016 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Interfaces including and related to IDistroSeries."""
@@ -91,6 +91,7 @@
     UniqueField,
     )
 from lp.services.webservice.apihelpers import patch_plain_parameter_type
+from lp.soyuz.enums import IndexCompressionType
 from lp.soyuz.interfaces.buildrecords import IHasBuildRecords
 from lp.translations.interfaces.hastranslationimports import (
     IHasTranslationImports,
@@ -378,6 +379,14 @@
                 on clients, which requires downloading Packages files for
                 multiple architectures.""")))
 
+    index_compressors = exported(List(
+        value_type=Choice(vocabulary=IndexCompressionType),
+        title=_("Compression types to use for published index files"),
+        required=True,
+        description=_("""
+            A list of compression types to use for published index files
+            (Packages, Sources, etc.).""")))
+
     inherit_overrides_from_parents = Bool(
         title=_("Inherit overrides from parents"),
         readonly=False, required=True)

=== modified file 'lib/lp/registry/model/distroseries.py'
--- lib/lp/registry/model/distroseries.py	2016-02-05 20:34:36 +0000
+++ lib/lp/registry/model/distroseries.py	2016-02-05 20:34:36 +0000
@@ -124,6 +124,7 @@
 from lp.services.worlddata.model.language import Language
 from lp.soyuz.enums import (
     ArchivePurpose,
+    IndexCompressionType,
     PackagePublishingStatus,
     PackageUploadStatus,
     )
@@ -202,6 +203,12 @@
     ]
 
 
+DEFAULT_INDEX_COMPRESSORS = [
+    IndexCompressionType.GZIP,
+    IndexCompressionType.BZIP2,
+    ]
+
+
 @implementer(
     IBugSummaryDimension, IDistroSeries, IHasBuildRecords, IHasQueueItems,
     IServiceUsage, ISeriesBugTarget)
@@ -266,6 +273,9 @@
             kwargs["publishing_options"] = {
                 "backports_not_automatic": False,
                 "include_long_descriptions": True,
+                "index_compressors": [
+                    compressor.title
+                    for compressor in DEFAULT_INDEX_COMPRESSORS],
                 }
         super(DistroSeries, self).__init__(*args, **kwargs)
 
@@ -816,6 +826,21 @@
         assert isinstance(value, bool)
         self.publishing_options["include_long_descriptions"] = value
 
+    @property
+    def index_compressors(self):
+        if "index_compressors" in self.publishing_options:
+            return [
+                IndexCompressionType.getTermByToken(name).value
+                for name in self.publishing_options["index_compressors"]]
+        else:
+            return list(DEFAULT_INDEX_COMPRESSORS)
+
+    @index_compressors.setter
+    def index_compressors(self, value):
+        assert isinstance(value, list)
+        self.publishing_options["index_compressors"] = [
+            compressor.title for compressor in value]
+
     def _customizeSearchParams(self, search_params):
         """Customize `search_params` for this distribution series."""
         search_params.setDistroSeries(self)

=== modified file 'lib/lp/registry/tests/test_distroseries.py'
--- lib/lp/registry/tests/test_distroseries.py	2015-12-15 14:12:25 +0000
+++ lib/lp/registry/tests/test_distroseries.py	2016-02-05 20:34:36 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2015 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2016 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Tests for distroseries."""
@@ -29,6 +29,7 @@
 from lp.services.webapp.interfaces import OAuthPermission
 from lp.soyuz.enums import (
     ArchivePurpose,
+    IndexCompressionType,
     PackagePublishingStatus,
     )
 from lp.soyuz.interfaces.archive import IArchiveSet
@@ -369,6 +370,19 @@
         self.assertFalse(
             naked_distroseries.publishing_options["include_long_descriptions"])
 
+    def test_index_compressors(self):
+        distroseries = self.factory.makeDistroSeries()
+        self.assertEqual(
+            [IndexCompressionType.GZIP, IndexCompressionType.BZIP2],
+            distroseries.index_compressors)
+        with admin_logged_in():
+            distroseries.index_compressors = [IndexCompressionType.XZ]
+        self.assertEqual(
+            [IndexCompressionType.XZ], distroseries.index_compressors)
+        naked_distroseries = removeSecurityProxy(distroseries)
+        self.assertEqual(
+            ["xz"], naked_distroseries.publishing_options["index_compressors"])
+
 
 class TestDistroSeriesPackaging(TestCaseWithFactory):
 

=== modified file 'lib/lp/soyuz/enums.py'
--- lib/lp/soyuz/enums.py	2015-09-03 15:14:07 +0000
+++ lib/lp/soyuz/enums.py	2016-02-05 20:34:36 +0000
@@ -1,4 +1,4 @@
-# Copyright 2010-2015 Canonical Ltd.  This software is licensed under the
+# Copyright 2010-2016 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Enumerations used in the lp/soyuz modules."""
@@ -13,6 +13,7 @@
     'archive_suffixes',
     'BinaryPackageFileType',
     'BinaryPackageFormat',
+    'IndexCompressionType',
     'PackageCopyPolicy',
     'PackageCopyStatus',
     'PackageDiffStatus',
@@ -570,3 +571,16 @@
         Specifies a native package, with a single tar.*. Supports gzip,
         bzip2, and xz compression.
         """)
+
+
+class IndexCompressionType(DBEnumeratedType):
+    """Index compression type
+
+    Archive indexes such as Packages and Sources may be compressed using any
+    of several different schemes.
+    """
+
+    UNCOMPRESSED = DBItem(0, "uncompressed")
+    GZIP = DBItem(1, "gzip")
+    BZIP2 = DBItem(2, "bzip2")
+    XZ = DBItem(3, "xz")

=== modified file 'setup.py'
--- setup.py	2015-11-04 14:30:09 +0000
+++ setup.py	2016-02-05 20:34:36 +0000
@@ -29,6 +29,7 @@
         'ampoule',
         'auditorclient',
         'auditorfixture',
+        'backports.lzma',
         'BeautifulSoup',
         'bzr',
         'cssselect',

=== modified file 'versions.cfg'
--- versions.cfg	2015-12-17 14:51:27 +0000
+++ versions.cfg	2016-02-05 20:34:36 +0000
@@ -15,6 +15,7 @@
 auditor = 0.0.3
 auditorclient = 0.0.4
 auditorfixture = 0.0.5
+backports.lzma = 0.0.3
 BeautifulSoup = 3.2.1
 billiard = 3.3.0.20
 bson = 0.3.3