cloud-init-dev team mailing list archive

[Wesley Wiedenmeier <wesley.wiedenmeier@xxxxxxxxx>]

I think launchpad's diff generator is broken so here's the diff:



diff --git a/tests/cloud_tests/args.py b/tests/cloud_tests/args.py
index b68cc98..44b2fb5 100644
--- a/tests/cloud_tests/args.py
+++ b/tests/cloud_tests/args.py
@@ -61,8 +61,12 @@ ARG_SETS = {
          {'help': 'ppa to enable (implies -u)', 'metavar': 'NAME',
           'action': 'store'}),
         (('-u', '--upgrade'),
-         {'help': 'upgrade before starting tests', 'action': 'store_true',
-          'default': False}),),
+         {'help': 'upgrade cloud-init from repo', 'action': 'store_true',
+          'default': False}),
+        (('--upgrade-full',),
+         {'help': 'do full system upgrade from repo (implies -u)',
+          'action': 'store_true', 'default': False}),),
+
 }
 
 SUBCMDS = {
diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py
index 68b47d7..032bdb0 100644
--- a/tests/cloud_tests/collect.py
+++ b/tests/cloud_tests/collect.py
@@ -18,8 +18,9 @@ def collect_script(instance, base_dir, script, script_name):
     return_value: None, may raise errors
     """
     LOG.debug('running collect script: %s', script_name)
-    util.write_file(os.path.join(base_dir, script_name),
-                    instance.run_script(script))
+    out = instance.run_script(script, ignore_errors=True,
+                              description='collect: {}'.format(script_name))
+    util.write_file(os.path.join(base_dir, script_name), out)
 
 
 def collect_test_data(args, snapshot, os_name, test_name):
diff --git a/tests/cloud_tests/images/base.py b/tests/cloud_tests/images/base.py
index 394b11f..cb1622e 100644
--- a/tests/cloud_tests/images/base.py
+++ b/tests/cloud_tests/images/base.py
@@ -28,10 +28,7 @@ class Image(object):
         """
         raise NotImplementedError
 
-    # FIXME: instead of having execute and push_file and other instance methods
-    #        here which pass through to a hidden instance, it might be better
-    #        to expose an instance that the image can be modified through
-    def execute(self, command, stdin=None, stdout=None, stderr=None, env={}):
+    def execute(self, *args, **kwargs):
         """
         execute command in image, modifying image
         """
@@ -43,7 +40,7 @@ class Image(object):
         """
         raise NotImplementedError
 
-    def run_script(self, script):
+    def run_script(self, *args, **kwargs):
         """
         run script in image, modifying image
         return_value: script output
diff --git a/tests/cloud_tests/images/lxd.py b/tests/cloud_tests/images/lxd.py
index 7a41614..261ba95 100644
--- a/tests/cloud_tests/images/lxd.py
+++ b/tests/cloud_tests/images/lxd.py
@@ -58,12 +58,12 @@ class LXDImage(base.Image):
         """
         return self.instance.push_file(local_path, remote_path)
 
-    def run_script(self, script):
+    def run_script(self, *args, **kwargs):
         """
         run script in image, modifying image
         return_value: script output
         """
-        return self.instance.run_script(script)
+        return self.instance.run_script(*args, **kwargs)
 
     def snapshot(self):
         """
diff --git a/tests/cloud_tests/instances/base.py b/tests/cloud_tests/instances/base.py
index 9559d28..8f8788b 100644
--- a/tests/cloud_tests/instances/base.py
+++ b/tests/cloud_tests/instances/base.py
@@ -16,11 +16,14 @@ class Instance(object):
         """
         self.name = name
 
-    def execute(self, command, stdin=None, stdout=None, stderr=None, env={}):
+    def execute(self, command, stdout=None, stderr=None, env={},
+                ignore_errors=False, description=None):
         """
         command: the command to execute as root inside the image
         stdin, stderr, stdout: file handles
         env: environment variables
+        ignore_errors: do not raise an error if the command fails
+        description: purpose of command
 
         Execute assumes functional networking and execution as root with the
         target filesystem being available at /.
@@ -60,13 +63,19 @@ class Instance(object):
         with open(local_path, 'rb') as fp:
             self.write_data(remote_path, fp.read())
 
-    def run_script(self, script):
+    def run_script(self, script, ignore_errors=False, description=None):
         """
         run script in target and return stdout
+        script: script contents
+        ignore_errors: do not raise an error if the script returns non-zero
+        description: purpose of script
+        return_value: stdout from script
         """
         script_path = os.path.join('/tmp', str(uuid.uuid1()))
         self.write_data(script_path, script)
-        (out, err, exit_code) = self.execute(['/bin/bash', script_path])
+        (out, err, exit_code) = self.execute(['/bin/bash', script_path],
+                                             ignore_errors=ignore_errors,
+                                             description=description)
         return out
 
     def console_log(self):
diff --git a/tests/cloud_tests/instances/lxd.py b/tests/cloud_tests/instances/lxd.py
index f0aa121..24a516e 100644
--- a/tests/cloud_tests/instances/lxd.py
+++ b/tests/cloud_tests/instances/lxd.py
@@ -1,6 +1,7 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 from tests.cloud_tests.instances import base
+from tests.cloud_tests import util
 
 
 class LXDInstance(base.Instance):
@@ -22,28 +23,31 @@ class LXDInstance(base.Instance):
         self._pylxd_container.sync()
         return self._pylxd_container
 
-    def execute(self, command, stdin=None, stdout=None, stderr=None, env={}):
+    def execute(self, command, stdout=None, stderr=None, env={},
+                ignore_errors=False, description=None):
         """
         command: the command to execute as root inside the image
-        stdin, stderr, stdout: file handles
+        stderr, stdout: file handles to write output to
         env: environment variables
+        ignore_errors: do not raise an error if the command fails
+        description: purpose of command
 
         Execute assumes functional networking and execution as root with the
         target filesystem being available at /.
 
         return_value: tuple containing stdout data, stderr data, exit code
         """
-        # TODO: the pylxd api handler for container.execute needs to be
-        #       extended to properly pass in stdin
-        # TODO: the pylxd api handler for container.execute needs to be
-        #       extended to get the return code, for now just use 0
         self.start()
-        if stdin:
-            raise NotImplementedError
-        res = self.pylxd_container.execute(command, environment=env)
-        for (f, data) in (i for i in zip((stdout, stderr), res) if i[0]):
-            f.write(data)
-        return res + (0,)
+        exit, out, err = self.pylxd_container.execute(command, environment=env)
+        for (fp, data) in (i for i in zip((stdout, stderr), (out, err))
+                           if i[0] and getattr(i, 'writable', False)):
+            fp.write(data)
+        if exit != 0 and not ignore_errors:
+            error_desc = ('Failed command to: {}'.format(description)
+                          if description else None)
+            raise util.InTargetExecuteError(out, err, exit, command, self.name,
+                                            error_desc)
+        return (out, err, exit)
 
     def read_data(self, remote_path, decode=False):
         """
diff --git a/tests/cloud_tests/setup_image.py b/tests/cloud_tests/setup_image.py
index 5d6c638..18b68b9 100644
--- a/tests/cloud_tests/setup_image.py
+++ b/tests/cloud_tests/setup_image.py
@@ -7,6 +7,30 @@ from functools import partial
 import os
 
 
+def installed_version(image, package, ensure_installed=True):
+    """
+    get installed version of package
+    image: cloud_tests.images instance to operate on
+    package: name of package
+    ensure_installed: raise error if not installed
+    return_value: cloud-init version string
+    """
+    # get right cmd for os family
+    os_family = util.get_os_family(image.properties['os'])
+    if os_family == 'debian':
+        cmd = ['dpkg-query', '-W', "--showformat='${Version}'", package]
+    elif os_family == 'redhat':
+        cmd = ['rpm', '-q', '--queryformat', "'%{VERSION}'", package]
+    else:
+        raise NotImplementedError
+
+    # query version
+    msg = 'query version for package: {}'.format(package)
+    (out, err, exit) = image.execute(cmd, description=msg,
+                                     ignore_errors=not ensure_installed)
+    return out.strip()
+
+
 def install_deb(args, image):
     """
     install deb into image
@@ -21,20 +45,17 @@ def install_deb(args, image):
                                   'family: {}'.format(args.deb, os_family))
 
     # install deb
-    LOG.debug('installing deb: %s into target', args.deb)
+    msg = 'install deb: "{}" into target'.format(args.deb)
+    LOG.debug(msg)
     remote_path = os.path.join('/tmp', os.path.basename(args.deb))
     image.push_file(args.deb, remote_path)
-    (out, err, exit) = image.execute(['dpkg', '-i', remote_path])
-    if exit != 0:
-        raise OSError('failed install deb: {}\n\tstdout: {}\n\tstderr: {}'
-                      .format(args.deb, out, err))
+    image.execute(['dpkg', '-i', remote_path], description=msg)
 
     # check installed deb version matches package
     fmt = ['-W', "--showformat='${Version}'"]
     (out, err, exit) = image.execute(['dpkg-deb'] + fmt + [remote_path])
     expected_version = out.strip()
-    (out, err, exit) = image.execute(['dpkg-query'] + fmt + ['cloud-init'])
-    found_version = out.strip()
+    found_version = installed_version(image, 'cloud-init')
     if expected_version != found_version:
         raise OSError('install deb version "{}" does not match expected "{}"'
                       .format(found_version, expected_version))
@@ -57,19 +78,16 @@ def install_rpm(args, image):
                                   'family: {}'.format(args.rpm, os_family))
 
     # install rpm
-    LOG.debug('installing rpm: %s into target', args.rpm)
+    msg = 'install rpm: "{}" into target'.format(args.rpm)
+    LOG.debug(msg)
     remote_path = os.path.join('/tmp', os.path.basename(args.rpm))
     image.push_file(args.rpm, remote_path)
-    (out, err, exit) = image.execute(['rpm', '-U', remote_path])
-    if exit != 0:
-        raise OSError('failed to install rpm: {}\n\tstdout: {}\n\tstderr: {}'
-                      .format(args.rpm, out, err))
+    image.execute(['rpm', '-U', remote_path], description=msg)
 
     fmt = ['--queryformat', '"%{VERSION}"']
     (out, err, exit) = image.execute(['rpm', '-q'] + fmt + [remote_path])
     expected_version = out.strip()
-    (out, err, exit) = image.execute(['rpm', '-q'] + fmt + ['cloud-init'])
-    found_version = out.strip()
+    found_version = installed_version(image, 'cloud-init')
     if expected_version != found_version:
         raise OSError('install rpm version "{}" does not match expected "{}"'
                       .format(found_version, expected_version))
@@ -80,13 +98,34 @@ def install_rpm(args, image):
 
 def upgrade(args, image):
     """
-    run the system's upgrade command
+    upgrade cloud-init from repo
+    args: cmdline arguments
+    image: cloud_tests.images instance to operate on
+    return_value: None, may raise errors
+    """
+    # determine command for os_family
+    os_family = util.get_os_family(image.properties['os'])
+    if os_family == 'debian':
+        cmd = 'apt-get update && apt-get install cloud-init --yes'
+    elif os_family == 'redhat':
+        cmd = 'yum update cloud-init --assumeyes'
+    else:
+        raise NotImplementedError
+
+    # upgrade cloud-init
+    msg = 'upgrading cloud-init'
+    LOG.debug(msg)
+    image.execute(['/bin/sh', '-c', cmd], description=msg)
+
+
+def upgrade_full(args, image):
+    """
+    run the system's full upgrade command
     args: cmdline arguments
     image: cloud_tests.images instance to operate on
     return_value: None, may raise errors
     """
     # determine appropriate upgrade command for os_family
-    # TODO: maybe use cloudinit.distros for this?
     os_family = util.get_os_family(image.properties['os'])
     if os_family == 'debian':
         cmd = 'apt-get update && apt-get upgrade --yes'
@@ -97,11 +136,9 @@ def upgrade(args, image):
                                   'from family: {}'.format(os_family))
 
     # upgrade system
-    LOG.debug('upgrading system')
-    (out, err, exit) = image.execute(['/bin/sh', '-c', cmd])
-    if exit != 0:
-        raise OSError('failed to upgrade system\n\tstdout: {}\n\tstderr:{}'
-                      .format(out, err))
+    msg = 'full system upgrade'
+    LOG.debug(msg)
+    image.execute(['/bin/sh', '-c', cmd], description=msg)
 
 
 def run_script(args, image):
@@ -111,9 +148,9 @@ def run_script(args, image):
     image: cloud_tests.images instance to operate on
     return_value: None, may raise errors
     """
-    # TODO: get exit status back from script and add error handling here
-    LOG.debug('running setup image script in target image')
-    image.run_script(args.script)
+    msg = 'run setup image script in target image'
+    LOG.debug(msg)
+    image.run_script(args.script, description=msg)
 
 
 def enable_ppa(args, image):
@@ -129,12 +166,10 @@ def enable_ppa(args, image):
 
     # add ppa with add-apt-repository and update
     ppa = 'ppa:{}'.format(args.ppa)
-    LOG.debug('enabling %s', ppa)
+    msg = 'enable ppa: "{}" in target'.format(ppa)
+    LOG.debug(msg)
     cmd = 'add-apt-repository --yes {} && apt-get update'.format(ppa)
-    (out, err, exit) = image.execute(['/bin/sh', '-c', cmd])
-    if exit != 0:
-        raise OSError('enable ppa for {} failed\n\tstdout: {}\n\tstderr: {}'
-                      .format(ppa, out, err))
+    image.execute(['/bin/sh', '-c', cmd], description=msg)
 
 
 def enable_repo(args, image):
@@ -155,11 +190,9 @@ def enable_repo(args, image):
         raise NotImplementedError('enable repo command not configured for '
                                   'distro from family: {}'.format(os_family))
 
-    LOG.debug('enabling repo: "%s"', args.repo)
-    (out, err, exit) = image.execute(['/bin/sh', '-c', cmd])
-    if exit != 0:
-        raise OSError('enable repo {} failed\n\tstdout: {}\n\tstderr: {}'
-                      .format(args.repo, out, err))
+    msg = 'enable repo: "{}" in target'.format(args.repo)
+    LOG.debug(msg)
+    image.execute(['/bin/sh', '-c', cmd], description=msg)
 
 
 def setup_image(args, image):
@@ -179,7 +212,8 @@ def setup_image(args, image):
         ('repo', enable_repo, 'setup func for --repo, enable repo'),
         ('ppa', enable_ppa, 'setup func for --ppa, enable ppa'),
         ('script', run_script, 'setup func for --script, run script'),
-        ('upgrade', upgrade, 'setup func for --upgrade, upgrade pkgs'),
+        ('upgrade', upgrade, 'setup func for --upgrade, upgrade cloud-init'),
+        ('upgrade-full', upgrade_full, 'setup func for --upgrade-full'),
     )
 
     # determine which setup functions needed
@@ -189,7 +223,10 @@ def setup_image(args, image):
     image_name = 'image: distro={}, release={}'.format(
         image.properties['os'], image.properties['release'])
     LOG.info('setting up %s', image_name)
-    return stage.run_stage('set up for {}'.format(image_name), calls,
-                           continue_after_error=False)
+    res = stage.run_stage('set up for {}'.format(image_name), calls,
+                          continue_after_error=False)
+    LOG.debug('after setup complete, installed cloud-init version is: %s',
+              installed_version(image, 'cloud-init'))
+    return res
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/util.py b/tests/cloud_tests/util.py
index 64a8667..5fa1f40 100644
--- a/tests/cloud_tests/util.py
+++ b/tests/cloud_tests/util.py
@@ -160,4 +160,23 @@ def write_file(*args, **kwargs):
     """
     c_util.write_file(*args, **kwargs)
 
+
+class InTargetExecuteError(c_util.ProcessExecutionError):
+    """
+    Error type for in target commands that fail
+    """
+    default_desc = 'Unexpected error while running command in target instance'
+
+    def __init__(self, stdout, stderr, exit_code, cmd, instance,
+                 description=None):
+        """
+        init error and parent error class
+        """
+        if isinstance(cmd, (tuple, list)):
+            cmd = ' '.join(cmd)
+        super(InTargetExecuteError, self).__init__(
+            stdout=stdout, stderr=stderr, exit_code=exit_code, cmd=cmd,
+            reason="Instance: {}".format(instance),
+            description=description if description else self.default_desc)
+
 # vi: ts=4 expandtab

-- 
https://code.launchpad.net/~wesley-wiedenmeier/cloud-init/+git/cloud-init/+merge/313871
Your team cloud init development team is requested to review the proposed merge of ~wesley-wiedenmeier/cloud-init:integration-testing-merge-update into cloud-init:master.