canonical-ubuntu-qa team mailing list archive

[Tim Andersson <mp+488013@xxxxxxxxxxxxxxxxxx>]

Tim Andersson has proposed merging ~andersson123/utah:preseed-runlist-url-checks into utah:master.

Requested reviews:
  UTAH Dev (utah)

For more details, see:
https://code.launchpad.net/~andersson123/utah/+git/utah/+merge/488013

Drop some command line arg checks from utah to make it work with git.lp.n
-- 
Your team UTAH Dev is requested to review the proposed merge of ~andersson123/utah:preseed-runlist-url-checks into utah:master.

diff --git a/utah/client/vcs.py b/utah/client/vcs.py
index 912b741..f559943 100644
--- a/utah/client/vcs.py
+++ b/utah/client/vcs.py
@@ -18,6 +18,7 @@
 import logging
 import os
 import subprocess
+import shutil
 
 from utah.client.common import (
     CMD_TS_FETCH,
@@ -128,40 +129,73 @@ class GitHandler(VCSHandler):
 
     """Git VCS handler."""
 
-    def __init__(self, repo, options="", destination="", **kwargs):
+    def __init__(self, repo, sparse=True, squash_to_root_dir=False, options="", destination="", **kwargs):
+        # This hacky string splitting is done in an effort to have minimal diffs in utah, since it's
+        # in maintenance mode.
+        # Anything else would require schema changes and change the way utah parses .run files.
+        # So, instead of introducing any mess on that front, let's just use a special character
+        # to split up the fetch_location value.
+        if "$" in repo:
+            assert len(repo.split("$")) == 3
+            repo, self.branch, path = repo.split("$")
+            self.path = path
+        else:
+            self.branch = "master"
+            self.path = ""
         super(GitHandler, self).__init__(repo, destination, **kwargs)
         self.options = options
+        self.squash_to_root_dir = squash_to_root_dir
 
-        self.get_command = "git clone {} {} {}".format(
-            options,
-            self.repo,
-            self.destination,
-        )
+        if sparse:
+            self.get_command = "git clone {} -b {} -n --depth=1 --filter=tree:0 {} {}".format(
+                options,
+                self.branch,
+                self.repo,
+                self.destination,
+            )
+        else:
+            self.get_command = "git clone {} {} {}".format(
+                options,
+                self.repo,
+                self.destination,
+            )
 
         self.rev_command = "git rev-parse HEAD"
 
-    def sparse_checkout(self, paths):
+    def sparse_checkout(self, path):
         """
         Do a sparse checkout in the destination folder
-        :param paths: the repository path to be added in the checkout
+        :param path: the repository path to be added in the sparse checkout
         """
         current_dir = os.getcwd()
-        os.chdir(self.destination)
+        # On the testbed, this corresponds to /var/lib/utah/default/default
+        os.chdir(os.path.join(self.destination, self.destination))
         try:
-            subprocess.check_call(['git', 'init'])
-            subprocess.check_call(['git', 'remote', 'add', '-f', 'origin',
-                                   self.repo])
-            subprocess.check_call(['git', 'config', 'core.sparseCheckout',
-                                   'true'])
-            with open('.git/info/sparse-checkout', 'w') as f:
-                for path in paths:
-                    f.write(path)
-            self.pull()
+            subprocess.check_call(["git", "sparse-checkout", "set", "--no-cone", path])
+            subprocess.check_call(["git", "checkout"])
+            if self.squash_to_root_dir:
+                # we must copy the contents of repo_root/path/to/sparse to repo_root, to emulate
+                # the behaviour of bzr export default repo_root/path/to/sparse. a sparse checkout
+                # with git preserves the nested directories, whereas a bzr export does not.
+                # e.g., we copy the contents of default/testsuites/default/ to default/, then we
+                # recursively remove the testsuites directory, so everything that existed in
+                # testsuites/default is now in the root of the repo.
+                # We could change the default paths of the .run and .preseed files, but then that'd make
+                # the BzrHandler dysfunctional.
+                shutil.copytree(path, os.getcwd(), dirs_exist_ok=True)
+                shutil.rmtree(path.split("/")[0])
         except subprocess.CalledProcessError:
             raise
         finally:
             os.chdir(current_dir)
 
+    def get(self, directory=REPO_DEFAULT_DIR):
+        res = super(GitHandler, self).get(directory)
+        if res['returncode'] == 3 and self._retriable(res):
+            msg = 'Launchpad temporary error detected: {}'.format(res)
+            raise UTAHException(msg, retry=True)
+        self.sparse_checkout(self.path if self.path else directory)
+
     def pull(self, repo='origin', branch='master'):
         current_dir = os.getcwd()
         os.chdir(self.destination)
@@ -215,7 +249,7 @@ def create_vcs_handler(method, location, destination, runlist):
         'bzr': (BzrHandler, {}),
         'bzr-export': (BzrHandler, {'branch': False}),
         'dev': (DevHandler, {}),
-        'git': (GitHandler, {}),
+        'git': (GitHandler, {'squash_to_root_dir': True}),
     }
 
     cls, args = handlers[method]
diff --git a/utah/isotest/iso_static_validation.py b/utah/isotest/iso_static_validation.py
index c88bc21..2073961 100755
--- a/utah/isotest/iso_static_validation.py
+++ b/utah/isotest/iso_static_validation.py
@@ -568,7 +568,7 @@ class TestValidateISO(unittest.TestCase):
             # so we just pull to ensure it is up to date
             git_keys.pull()
         else:
-            git_keys.sparse_checkout(paths=[keys_repo_path])
+            git_keys.get(directory=keys_repo_path)
         cmd = ["make", "-C", keys]
         output = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
diff --git a/utah/parser.py b/utah/parser.py
index aeca2ad..401dd4a 100644
--- a/utah/parser.py
+++ b/utah/parser.py
@@ -20,7 +20,6 @@ import sys
 import os
 
 from utah.config import config
-from utah.run import master_runlist_argument
 from utah.url import url_argument
 
 
@@ -96,7 +95,7 @@ def get_parser():
                         help='Reuse a system that is already provisioned '
                              '(name argument must be passed)')
     parser.add_argument('runlist', metavar='runlist',
-                        type=master_runlist_argument,
+                        type=str,
                         help='URLs of runlist files to run')
     parser.add_argument('-s', '--series', metavar='SERIES',
                         choices=config.serieschoices,
@@ -135,7 +134,7 @@ def get_parser():
     parser.add_argument('-i', '--image', type=url_argument,
                         default=config.image,
                         help='Image/ISO file to use for installation')
-    parser.add_argument('-p', '--preseed', type=url_argument,
+    parser.add_argument('-p', '--preseed', type=str,
                         default=config.preseed,
                         help='Preseed file to use for installation')
     parser.add_argument('-b', '--boot',