curtin-dev team mailing list archive

[Dan Bungert <mp+443376@xxxxxxxxxxxxxxxxxx>]

Dan Bungert has proposed merging ~dbungert/curtin:test-help into curtin:master.

Commit message:
fixes for running the unit and integration tests

Update some out-of-date build and test infrastructure

Requested reviews:
  curtin developers (curtin-dev)

For more details, see:
https://code.launchpad.net/~dbungert/curtin/+git/curtin/+merge/443376
-- 
Your team curtin developers is requested to review the proposed merge of ~dbungert/curtin:test-help into curtin:master.

diff --git a/Makefile b/Makefile
index c04c4cf..5e8822a 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,5 @@
 TOP := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
 CWD := $(shell pwd)
-PYTHON2 ?= python2
 PYTHON3 ?= python3
 COVERAGE ?= 1
 DEFAULT_COVERAGEOPTS = --with-coverage --cover-erase --cover-branches --cover-package=curtin --cover-inclusive 
@@ -16,11 +15,11 @@ target_dirs ?= curtin tests tools
 build:
 
 bin/curtin: curtin/pack.py tools/write-curtin
-	$(PYTHON) tools/write-curtin bin/curtin
+	$(PYTHON3) tools/write-curtin bin/curtin
 
 check: unittest
 
-style-check: pep8 pyflakes pyflakes3
+style-check: pep8 pyflakes3
 
 coverage: coverageopts ?= $(DEFAULT_COVERAGEOPTS)
 coverage: unittest
@@ -28,26 +27,15 @@ coverage: unittest
 pep8:
 	@$(CWD)/tools/run-pep8
 
-pyflakes:
-	$(PYTHON2) -m pyflakes $(target_dirs)
-
-pyflakes3:
+pyflakes pyflakes3:
 	$(PYTHON3) -m pyflakes $(target_dirs)
 
-pylint:
-	$(PYTHON2) -m pylint $(pylintopts) $(target_dirs)
-
-pylint3:
+pylint pylint3:
 	$(PYTHON3) -m pylint $(pylintopts) $(target_dirs)
 
-unittest2:
-	$(PYTHON2) -m nose $(coverageopts) $(noseopts) tests/unittests
-
-unittest3:
+unittest unittest3:
 	$(PYTHON3) -m nose $(coverageopts) $(noseopts) tests/unittests
 
-unittest: unittest2 unittest3
-
 schema-validate:
 	@$(CWD)/tools/schema-validate-storage
 
@@ -55,7 +43,7 @@ docs: check-doc-deps
 	make -C doc html
 
 check-doc-deps:
-	@which sphinx-build && $(PYTHON) -c 'import sphinx_rtd_theme' || \
+	@which sphinx-build && $(PYTHON3) -c 'import sphinx_rtd_theme' || \
 		{ echo "Missing doc dependencies. Install with:"; \
 		  pkgs="python3-sphinx-rtd-theme python3-sphinx"; \
 		  echo sudo apt-get install -qy $$pkgs ; exit 1; }
diff --git a/requirements.txt b/requirements.txt
index d8d1a3d..71a2637 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,6 @@
+pip
+wheel
+virtualenv
 pyudev
 pyyaml
 oauthlib
diff --git a/test-requirements.txt b/test-requirements.txt
index 1970d03..0d93c02 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -1,3 +1,6 @@
+pip
+wheel
+virtualenv
 jsonschema
 mock
 nose
diff --git a/tests/integration/test_block_meta.py b/tests/integration/test_block_meta.py
index d701ca0..d14a3f8 100644
--- a/tests/integration/test_block_meta.py
+++ b/tests/integration/test_block_meta.py
@@ -8,6 +8,7 @@ import os
 from parameterized import parameterized
 import re
 import sys
+import tempfile
 from typing import Optional
 import yaml
 
@@ -168,8 +169,7 @@ class StorageConfigBuilder:
 
     def add_image(self, *, path, size, create=False, **kw):
         if create:
-            with open(path, "wb") as f:
-                f.write(b"\0" * int(util.human2bytes(size)))
+            util.subp(['truncate', '-s', str(size), path])
         action = self._add(type='image', path=path, size=size, **kw)
         self.cur_image = action['id']
         return action
@@ -257,7 +257,25 @@ class TestBlockMeta(IntegrationTestCase):
             '-c', config_path, 'block-meta', '--testmode', 'custom',
             *args,
             ]
-        util.subp(cmd, env=cmd_env, **kwargs)
+
+        # Set debug=True to halt the integration test and run curtin manually,
+        # with the integration tests having setup the environment for you.
+        # Run tests with pytest-3 -s option.
+        if kwargs.pop('debug', False):
+            with tempfile.NamedTemporaryFile(mode='w', delete=False) as fp:
+                fp.write('#!/bin/bash\n')
+                for k, v in cmd_env.items():
+                    fp.write('export {}="{}"\n'.format(k, v))
+                fp.write('export PYTHONPATH="{}"\n'.format(os.getcwd()))
+                fp.write(' '.join(cmd) + '\n')
+            os.chmod(fp.name, 0o700)
+            print()
+            print('The integration test is paused.')
+            print('Use script {} to run curtin manually.'.format(fp.name))
+            breakpoint()
+            os.unlink(fp.name)
+        else:
+            util.subp(cmd, env=cmd_env, **kwargs)
 
     def _test_default_offsets(self, ptable, version, sector_size=512):
         psize = 40 << 20
diff --git a/tools/vmtest-system-setup b/tools/vmtest-system-setup
index 4596df5..c34e9aa 100755
--- a/tools/vmtest-system-setup
+++ b/tools/vmtest-system-setup
@@ -11,27 +11,37 @@ case "$(uname -m)" in
   s390x) qemu="qemu-system-s390x";;
 esac
 
-get_python_apt() {
-    [[ "$1" < "21.04" ]] && echo python-apt
-    [[ "$1" > "16.04" ]] && echo python3-apt
-}
-
 DEPS=(
+  build-essential
   cloud-image-utils
+  git
   make
   net-tools
+  pep8
   python3
+  python3-apt
   python3-attr
+  python3-coverage
   python3-jsonschema
   python3-nose
+  python3-oauthlib
+  python3-pip
+  python3-pyflakes
+  python3-pytest
+  python3-pyudev
   python3-simplestreams
+  python3-wheel
   python3-yaml
-  $(get_python_apt "$(lsb_release -sr)")
+  lvm2
+  ntfs-3g
   ovmf
+  parted
   simplestreams
   $qemu
   ubuntu-cloudimage-keyring
   tgt
+  tox
+  wget
 )
 
 apt_get() {