launchpad-reviewers team mailing list archive

[Colin Watson <cjwatson@xxxxxxxxxxxxx>]

Colin Watson has proposed merging lp:~cjwatson/launchpad/init-branch-livefses into lp:launchpad.

Commit message:
Copy live filesystems from the previous series on initialisation.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)
Related bugs:
  Bug #1387283 in Launchpad itself: "Please move branch-livefses into series initialization"
  https://bugs.launchpad.net/launchpad/+bug/1387283

For more details, see:
https://code.launchpad.net/~cjwatson/launchpad/init-branch-livefses/+merge/274277

Copy live filesystems from the previous series on initialisation.

To avoid zombie livefses hanging around forever, this only copies those that have had builds created for them in the last 30 days, so at worst they should exist for one series longer than their true lifetime.
-- 
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~cjwatson/launchpad/init-branch-livefses into lp:launchpad.

=== modified file 'lib/lp/soyuz/scripts/initialize_distroseries.py'
--- lib/lp/soyuz/scripts/initialize_distroseries.py	2015-05-13 12:40:44 +0000
+++ lib/lp/soyuz/scripts/initialize_distroseries.py	2015-10-13 15:40:46 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2014 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2015 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Initialize a distroseries from its parent distroseries."""
@@ -21,6 +21,7 @@
 from lp.registry.interfaces.pocket import PackagePublishingPocket
 from lp.registry.model.distroseries import DistroSeries
 from lp.services.database import bulk
+from lp.services.database.constants import THIRTY_DAYS_AGO
 from lp.services.database.interfaces import IMasterStore
 from lp.services.database.sqlbase import sqlvalues
 from lp.services.helpers import ensure_unicode
@@ -38,6 +39,7 @@
 from lp.soyuz.interfaces.distributionjob import (
     IDistroSeriesDifferenceJobSource,
     )
+from lp.soyuz.interfaces.livefs import ILiveFSSet
 from lp.soyuz.interfaces.packagecloner import IPackageCloner
 from lp.soyuz.interfaces.packageset import (
     IPackagesetSet,
@@ -45,6 +47,8 @@
     )
 from lp.soyuz.interfaces.queue import IPackageUploadSet
 from lp.soyuz.model.binarypackagebuild import COPY_ARCHIVE_SCORE_PENALTY
+from lp.soyuz.model.livefs import LiveFS
+from lp.soyuz.model.livefsbuild import LiveFSBuild
 from lp.soyuz.model.packageset import Packageset
 from lp.soyuz.scripts.packagecopier import do_copy
 
@@ -289,6 +293,7 @@
         self._copy_pocket_permissions()
         self._create_dsds()
         self._set_initialized()
+        self._copy_livefses()
         transaction.commit()
 
     def _set_parents(self):
@@ -737,3 +742,28 @@
                     """ % sqlvalues(
                         self.distroseries.main_archive, self.distroseries.id,
                         parent.id))
+
+    def _copy_livefses(self):
+        """Copy live filesystem configurations from the parent series."""
+        # We could probably be more efficient here, but the number of
+        # livefses is likely to be small.
+        livefs_set = getUtility(ILiveFSSet)
+        for parent in self.derivation_parents:
+            livefses = self._store.find(
+                LiveFS,
+                LiveFS.distro_series == parent,
+                LiveFSBuild.livefs == LiveFS.id,
+                LiveFSBuild.date_created >= THIRTY_DAYS_AGO)
+            for livefs in livefses:
+                if not livefs_set.exists(
+                        livefs.owner, self.distroseries, livefs.name):
+                    self._store.execute("""
+                        INSERT INTO LiveFS (
+                            registrant, owner, distro_series, name, json_data,
+                            require_virtualized)
+                        SELECT
+                            registrant, owner, %s, name, json_data,
+                            require_virtualized
+                        FROM LiveFS AS parent_livefs
+                        WHERE parent_livefs.id = %s
+                        """ % sqlvalues(self.distroseries.id, livefs.id))

=== modified file 'lib/lp/soyuz/scripts/tests/test_initialize_distroseries.py'
--- lib/lp/soyuz/scripts/tests/test_initialize_distroseries.py	2015-04-20 15:59:52 +0000
+++ lib/lp/soyuz/scripts/tests/test_initialize_distroseries.py	2015-10-13 15:40:46 +0000
@@ -5,6 +5,13 @@
 
 __metaclass__ = type
 
+from datetime import (
+    datetime,
+    timedelta,
+    )
+
+import pytz
+from testtools.matchers import MatchesStructure
 import transaction
 from zope.component import getUtility
 
@@ -19,7 +26,9 @@
     )
 from lp.registry.interfaces.distroseriesparent import IDistroSeriesParentSet
 from lp.registry.interfaces.pocket import PackagePublishingPocket
+from lp.services.database.constants import ONE_DAY_AGO
 from lp.services.database.interfaces import IStore
+from lp.services.features.testing import FeatureFixture
 from lp.soyuz.enums import (
     ArchivePurpose,
     PackageUploadStatus,
@@ -28,6 +37,10 @@
 from lp.soyuz.interfaces.archivepermission import IArchivePermissionSet
 from lp.soyuz.interfaces.binarypackagebuild import IBinaryPackageBuildSet
 from lp.soyuz.interfaces.component import IComponentSet
+from lp.soyuz.interfaces.livefs import (
+    ILiveFSSet,
+    LIVEFS_FEATURE_FLAG,
+    )
 from lp.soyuz.interfaces.packageset import (
     IPackagesetSet,
     NoSuchPackageSet,
@@ -1632,3 +1645,86 @@
             ("The selected architecture independent architecture tag is not "
              "among the selected architectures."),
             ids.check)
+
+    def test_copying_livefses(self):
+        # If a parent series has live filesystems, we should copy them.
+        self.parent, self.parent_das = self.setupParent()
+        with FeatureFixture({LIVEFS_FEATURE_FLAG: u'on'}):
+            livefs1 = self.factory.makeLiveFS(distroseries=self.parent)
+            self.factory.makeLiveFSBuild(
+                livefs=livefs1, date_created=ONE_DAY_AGO)
+            livefs2 = self.factory.makeLiveFS(
+                distroseries=self.parent,
+                metadata={'project': 'ubuntu-core', 'image_format': 'plain'},
+                require_virtualized=False)
+            self.factory.makeLiveFSBuild(
+                livefs=livefs2, date_created=ONE_DAY_AGO)
+        child = self._fullInitialize([self.parent])
+        child_livefs1 = getUtility(ILiveFSSet).getByName(
+            livefs1.owner, child, livefs1.name)
+        self.assertThat(child_livefs1, MatchesStructure.byEquality(
+            registrant=livefs1.registrant, owner=livefs1.owner,
+            distro_series=child, name=livefs1.name, metadata=livefs1.metadata,
+            require_virtualized=livefs1.require_virtualized))
+        child_livefs2 = getUtility(ILiveFSSet).getByName(
+            livefs2.owner, child, livefs2.name)
+        self.assertThat(child_livefs2, MatchesStructure.byEquality(
+            registrant=livefs2.registrant, owner=livefs2.owner,
+            distro_series=child, name=livefs2.name, metadata=livefs2.metadata,
+            require_virtualized=livefs2.require_virtualized))
+
+    def test_copying_livefses_no_duplication(self):
+        # Copying live filesystems only copies them from the most recent
+        # series, rather than merging them from all series.
+        previous_parent, _ = self.setupParent()
+        parent = self._fullInitialize([previous_parent])
+        with FeatureFixture({LIVEFS_FEATURE_FLAG: u'on'}):
+            parent_livefs1 = self.factory.makeLiveFS(
+                distroseries=parent, metadata={'marker': 'parent'})
+            self.factory.makeLiveFSBuild(
+                livefs=parent_livefs1, date_created=ONE_DAY_AGO)
+            parent_livefs2 = self.factory.makeLiveFS(distroseries=parent)
+            self.factory.makeLiveFSBuild(
+                livefs=parent_livefs2, date_created=ONE_DAY_AGO)
+        child = self._fullInitialize(
+            [previous_parent], previous_series=parent,
+            distribution=parent.distribution)
+        child_livefs1 = getUtility(ILiveFSSet).getByName(
+            parent_livefs1.owner, child, parent_livefs1.name)
+        child_livefs1.metadata = {'marker': 'child'}
+        with FeatureFixture({LIVEFS_FEATURE_FLAG: u'on'}):
+            self.factory.makeLiveFSBuild(
+                livefs=child_livefs1, date_created=ONE_DAY_AGO)
+        child_livefs2 = getUtility(ILiveFSSet).getByName(
+            parent_livefs2.owner, child, parent_livefs2.name)
+        child_livefs2.destroySelf()
+        grandchild = self._fullInitialize(
+            [previous_parent], previous_series=child,
+            distribution=parent.distribution)
+        grandchild_livefs1 = getUtility(ILiveFSSet).getByName(
+            child_livefs1.owner, grandchild, child_livefs1.name)
+        self.assertEqual({'marker': 'child'}, grandchild_livefs1.metadata)
+        self.assertFalse(
+            getUtility(ILiveFSSet).exists(
+                parent_livefs2.owner, grandchild, parent_livefs2.name))
+
+    def test_copying_livefses_skips_dormant(self):
+        # Live filesystems that have not been built in the last 30 days are
+        # not copied.
+        self.parent, self.parent_das = self.setupParent()
+        with FeatureFixture({LIVEFS_FEATURE_FLAG: u'on'}):
+            livefs1 = self.factory.makeLiveFS(distroseries=self.parent)
+            for days in 31, 30, 29:
+                self.factory.makeLiveFSBuild(
+                    livefs=livefs1,
+                    date_created=datetime.now(pytz.UTC) - timedelta(days=days))
+            livefs2 = self.factory.makeLiveFS(distroseries=self.parent)
+            for days in 33, 32, 31:
+                self.factory.makeLiveFSBuild(
+                    livefs=livefs2,
+                    date_created=datetime.now(pytz.UTC) - timedelta(days=days))
+        child = self._fullInitialize([self.parent])
+        self.assertTrue(
+            getUtility(ILiveFSSet).exists(livefs1.owner, child, livefs1.name))
+        self.assertFalse(
+            getUtility(ILiveFSSet).exists(livefs2.owner, child, livefs2.name))