launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #15168
[Merge] lp:~matsubara/maas/enable-cluster-controller into lp:~maas-maintainers/maas/qa-lab-tests
Diogo Matsubara has proposed merging lp:~matsubara/maas/enable-cluster-controller into lp:~maas-maintainers/maas/qa-lab-tests.
Commit message:
enable cluster controller tests
Requested reviews:
MAAS Maintainers (maas-maintainers)
For more details, see:
https://code.launchpad.net/~matsubara/maas/enable-cluster-controller/+merge/146202
This branch adds the necessary changes to test the cluster controller on a separate network. It enables the CC tests using an environment variable. I included some changes suggested by rvba to improve the code organization and remove some duplication.
A test run with this branch can be seen here: http://10.189.74.2:8080/view/CE/job/quantal-adt-maas-cluster-controller/ARCH=amd64,label=maas-lenovo-lab/143/console
--
https://code.launchpad.net/~matsubara/maas/enable-cluster-controller/+merge/146202
Your team MAAS Maintainers is requested to review the proposed merge of lp:~matsubara/maas/enable-cluster-controller into lp:~maas-maintainers/maas/qa-lab-tests.
=== added file 'cluster-controller-integration.py'
--- cluster-controller-integration.py 1970-01-01 00:00:00 +0000
+++ cluster-controller-integration.py 2013-02-14 13:55:23 +0000
@@ -0,0 +1,40 @@
+import os
+from testtools import TestCase
+from testtools.matchers import Contains
+from time import sleep
+
+from utils import run_command, update_pxe_config, SIGNAL_FILE
+
+
+class ClusterControllerIntegration(TestCase):
+
+ def test_01_preseed_updated_cluster_config(self):
+ # Make sure the cluster config was updated by the seed file.
+ maas_fd = open("/etc/maas/maas_cluster.conf" , "r+")
+ maas_file = maas_fd.read()
+ self.assertThat(maas_file, Contains(
+ 'MAAS_URL=http://192.168.21.5/MAAS'))
+
+ def test_02_update_pxe_config(self):
+ update_pxe_config()
+
+ def test_03_import_pxe_files(self):
+ output, err = run_command(["maas-import-pxe-files"])
+ self.assertThat(output, Contains('Downloading to temporary location'))
+ # XXX: matsubara Bug=1076444
+ # maas-import-pxe-files outputs to stderr during normal operation.
+ #self.assertIs(err, '')
+
+ def test_04_wait_for_region_controller(self):
+ """Wait for the region controller to run the integration tests.
+
+ The region controller will create a file in the cluster controller
+ once all tests finish.
+ """
+ while os.path.exists(SIGNAL_FILE) is False:
+ sleep(10)
+
+ @classmethod
+ def tearDownClass(cls):
+ """Power off the cluster controller VM after the test run."""
+ run_command(["sudo", "poweroff"])
=== removed file 'control'
--- control 2012-12-06 01:14:46 +0000
+++ control 1970-01-01 00:00:00 +0000
@@ -1,4 +0,0 @@
-Tests: maas-package-test
-# XXX: matsubara on precise you need to be explicit as @ is not being expanded
-# correctly
-Depends: @, maas, maas-dhcp, maas-dns, maas-cli, python-nose, bzr, python-testtools, python-yaml, juju
=== added file 'maas-cluster-controller-control'
--- maas-cluster-controller-control 1970-01-01 00:00:00 +0000
+++ maas-cluster-controller-control 2013-02-14 13:55:23 +0000
@@ -0,0 +1,2 @@
+Tests: maas-cluster-controller-package-test
+Depends: maas-cluster-controller, maas-dhcp, maas-cli, python-nose, bzr, python-testtools, python-yaml, juju, python-pip
=== added file 'maas-cluster-controller-package-test'
--- maas-cluster-controller-package-test 1970-01-01 00:00:00 +0000
+++ maas-cluster-controller-package-test 2013-02-14 13:55:23 +0000
@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e -u
+exec 2>&1
+pip install --no-index --find-links file:///home/ubuntu/ nose-timer
+nosetests --verbose --stop --with-timer debian/tests/cluster-controller-integration.py
=== added file 'maas-control'
--- maas-control 1970-01-01 00:00:00 +0000
+++ maas-control 2013-02-14 13:55:23 +0000
@@ -0,0 +1,2 @@
+Tests: maas-package-test
+Depends: maas, maas-dhcp, maas-dns, maas-cli, python-nose, bzr, python-testtools, python-yaml, juju, python-pip
=== modified file 'maas-integration.py'
--- maas-integration.py 2013-02-08 14:50:19 +0000
+++ maas-integration.py 2013-02-14 13:55:23 +0000
@@ -1,17 +1,16 @@
import os
from simplejson import loads
import sys
-from subprocess import Popen, PIPE
from time import sleep
from testtools import TestCase
-from testtools.matchers import Contains, Equals, StartsWith
+from testtools.matchers import Contains, StartsWith
from testtools.content import text_content
-from unittest import skipIf
+from unittest import skipIf, SkipTest
import yaml
import platform
import urllib2
-from timeout import timeout
+from utils import run_command, timeout, update_pxe_config, SIGNAL_FILE
sys.path.insert(0, "/usr/share/maas")
os.environ['DJANGO_SETTINGS_MODULE'] = 'maas.settings'
@@ -19,9 +18,11 @@
from django.contrib.auth.models import User
from maasserver.models.user import get_creds_tuple
+from maasserver.models import BootImage
+from maasserver.utils import get_local_cluster_UUID
from apiclient.creds import convert_tuple_to_string
-# Environment variables that can be used to configured
+# Environment variables that can be used to configure
# what is tested.
# Series to install on deployed nodes.
@@ -40,6 +41,8 @@
DO_NOT_TEST_JUJU = bool(
os.environ.get('DO_NOT_TEST_JUJU', False))
+# Whether or not the cluster controller nodes should be used.
+USE_CC_NODES = bool(os.environ.get('USE_CC_NODES', False))
MAAS_URL = "http://192.168.21.5/MAAS";
ADMIN_USER = "admin"
@@ -47,21 +50,28 @@
POWER_USER = "root"
POWER_PASS = "ubuntu"
-LENOVO_LAB = {
+REGION_CONTROLLER_NODES = {
"00:E0:81:DD:D5:99" : "192.168.22.33",
"00:E0:81:DD:D1:0B" : "192.168.22.34",
"00:E0:81:DD:D4:11" : "192.168.22.35",
"00:E0:81:D1:B1:47" : "192.168.22.36",
-# There are in a different network, enable them
-# when testing with a CC on a different network
-# is supported.
-# "00:E0:81:DD:D1:1B" : "192.168.22.37",
-# "00:E0:81:DD:D1:2B" : "192.168.22.38",
-# "00:E0:81:DD:D1:A3" : "192.168.22.39",
-# "00:E0:81:DC:38:6D" : "192.168.22.40",
-# "00:E0:81:DD:D0:FF" : "192.168.22.41",
-# "00:E0:81:DD:D4:F9" : "192.168.22.42"
- }
+ }
+
+CLUSTER_CONTROLLER_NODES = {
+ "00:E0:81:DD:D1:1B" : "192.168.22.37",
+ "00:E0:81:DD:D1:2B" : "192.168.22.38",
+ "00:E0:81:DD:D1:A3" : "192.168.22.39",
+ "00:E0:81:DC:38:6D" : "192.168.22.40",
+ "00:E0:81:DD:D0:FF" : "192.168.22.41",
+ "00:E0:81:DD:D4:F9" : "192.168.22.42"
+ }
+
+CLUSTER_CONTROLLER_IP = '192.168.20.5'
+
+if USE_CC_NODES:
+ LENOVO_LAB = dict(REGION_CONTROLLER_NODES, **CLUSTER_CONTROLLER_NODES)
+else:
+ LENOVO_LAB = REGION_CONTROLLER_NODES
ARM_LAB = {
'fc:2f:40:d8:fb:1a': '192.168.21.50',
@@ -123,9 +133,7 @@
os.mkdir(ssh_dir)
except OSError:
pass
- Popen(
- ['ssh-keygen', '-t', 'rsa', '-N', '', '-f', ssh_key],
- stdout=PIPE, stderr=PIPE).communicate()
+ run_command(['ssh-keygen', '-t', 'rsa', '-N', '', '-f', ssh_key])
# Setup ssh config.
ssh_config = os.path.join(ssh_dir, 'config')
with open(ssh_config, 'w') as f:
@@ -139,6 +147,7 @@
f.write(content)
+
DEB_PROXY_CONFIG = """
cache_peer 10.98.0.13 parent 8000 0 no-query no-digest
never_direct allow all
@@ -152,9 +161,7 @@
content = content + DEB_PROXY_CONFIG
with open(config, 'w') as f:
f.write(content)
- Popen(
- ['sudo', 'service', 'squid-deb-proxy', 'restart'],
- stdout=PIPE, stderr=PIPE).communicate()
+ run_command(['sudo', 'service', 'squid-deb-proxy', 'restart'])
class TestMAASIntegration(TestCase):
@@ -166,13 +173,8 @@
count += len(ARM_LAB)
return count
- def _run_command(self, args):
- process = Popen(args, stdout=PIPE, stderr=PIPE, stdin=PIPE)
- stdout, stderr = process.communicate()
- return stdout, stderr
-
def _run_maas_cli(self, args):
- result = self._run_command(["maas-cli", "maas"] + args)
+ result = run_command(["maas-cli", "maas"] + args)
self.addDetail(
'maas-cli maas %s' % str(args), text_content(str(result)))
return result
@@ -210,47 +212,22 @@
maas_fd.seek(0)
maas_fd.write(maas_file)
maas_fd.close()
- output, err = self._run_command(["service", "apache2", "restart"])
+ output, err = run_command(["service", "apache2", "restart"])
self.assertThat(
output, StartsWith(' * Restarting web server apache2'))
def test_03_restart_dbus_avahi(self):
"""XXX: matsubara bug=1065775 """
- output, err = self._run_command(["service", "dbus", "restart"])
+ output, err = run_command(["service", "dbus", "restart"])
self.assertThat(output, Contains('dbus start/running'))
- output, err = self._run_command(["service", "avahi-daemon", "restart"])
+ output, err = run_command(["service", "avahi-daemon", "restart"])
self.assertThat(output, Contains('avahi-daemon start/running'))
def test_04_update_pxe_config(self):
- ephemerals_fd = open("/etc/maas/import_ephemerals", "r+")
- ephemerals = ephemerals_fd.read()
- # Update mirrors for both trunk and the quantal SRU.
- ephemerals_snippet = (
- '\n'
- 'CLOUD_IMAGES_ARCHIVE="http://10.98.0.13/mirrors/maas-ephemeral"\n'
- 'REMOTE_IMAGES_MIRROR="http://10.98.0.13/mirrors/maas-ephemeral"\n'
- # XXX: rvb 2013-01-18: Do not use the daily ephemeral images as
- # they seem to be broken (more precisely, the image from 20130107 is
- # and the one from 20121008 isn't); investigation is underway.
- # 'STREAM=daily\n'
- )
- ephemerals += ephemerals_snippet
- ephemerals_fd.seek(0)
- ephemerals_fd.write(ephemerals)
- ephemerals_fd.close()
- # XXX: matsubara Bug=1074167
- # Update import_pxe_files to not download squashfs images
- # and use a proxy.
- pxe_fd = open('/etc/maas/import_pxe_files', "r+")
- pxe_file = pxe_fd.read()
- pxe_snippet = 'export http_proxy="http://10.98.0.13:3128"\n'
- pxe_file = pxe_snippet + pxe_file
- pxe_fd.seek(0)
- pxe_fd.write(pxe_file)
- pxe_fd.close()
+ update_pxe_config()
def test_05_import_pxe_files(self):
- output, err = self._run_command(["maas-import-pxe-files"])
+ output, err = run_command(["maas-import-pxe-files"])
self.assertThat(output, Contains('Downloading to temporary location'))
# XXX: matsubara Bug=1076444
# maas-import-pxe-files outputs to stderr during normal operation.
@@ -268,7 +245,7 @@
def test_07_login_api(self):
token_str = get_token_str()
api_url = MAAS_URL + "/api/1.0/"
- output, err = self._run_command(
+ output, err = run_command(
["maas-cli", "login", "maas", api_url, token_str])
self.assertThat(
output, Contains(
@@ -302,31 +279,76 @@
output, err = self._run_command(["service", "apparmor", "reload"])
self.assertThat(output, Contains('* Reloading AppArmor profiles'))
- @timeout(60)
+ def _set_up_dhcp(self, uuid, dhcp_config):
+ dhcp_cli_args = [
+ "%s=%s" % (key, value) for key, value in dhcp_config.items()]
+ maas_dhcp_cmd = [
+ "node-group-interface", "update", uuid, "eth1" ] + dhcp_cli_args
+ output, err = self._run_maas_cli(maas_dhcp_cmd)
+ node_group = loads(output)
+ # The JSON object returned by MAAS doesn't include the router_ip
+ # address, so let's remove it from the dhcp_config before comparing.
+ dhcp_config.pop('router_ip')
+ self.assertEquals(node_group, dhcp_config)
+
+ @timeout(5*60)
def test_09_set_up_dhcp(self):
if "raring" in platform.linux_distribution():
self._update_dhcpd_apparmor_profile()
+ region_uuid = get_local_cluster_UUID()
+
output, err = self._run_maas_cli(["node-groups", "list"])
node_groups = loads(output)
- output, err = self._run_maas_cli([
- "node-group-interface", "update", node_groups[0]['uuid'],
- "eth1", "ip=192.168.21.5", "interface=eth1", "management=2",
- "subnet_mask=255.255.255.0",
- "broadcast_ip=192.168.21.255",
- "router_ip=192.168.21.1",
- "ip_range_low=192.168.21.10",
- "ip_range_high=192.168.21.30"])
- node_group = loads(output)
- self.assertThat(node_group['ip'], Equals('192.168.21.5'))
- self.assertThat(node_group['interface'], Equals('eth1'))
- self.assertThat(node_group['subnet_mask'], Equals('255.255.255.0'))
- self.assertThat(node_group['broadcast_ip'], Equals('192.168.21.255'))
- self.assertThat(node_group['ip_range_low'], Equals('192.168.21.10'))
- self.assertThat(node_group['ip_range_high'], Equals('192.168.21.30'))
+
+ ng_uuids = [ng['uuid'] for ng in node_groups]
+ self.assertIn(region_uuid, ng_uuids)
+
+ region_dhcp_config = {
+ "ip": "192.168.21.5",
+ "interface": "eth1",
+ "subnet_mask": "255.255.255.0",
+ "broadcast_ip": "192.168.21.255",
+ "router_ip": "192.168.21.1",
+ "management": 2,
+ "ip_range_low": "192.168.21.10",
+ "ip_range_high": "192.168.21.30"}
+
+ self._set_up_dhcp(region_uuid, region_dhcp_config)
+
# Wait for the task to complete and create the dhcpd.conf file.
while os.path.exists("/etc/maas/dhcpd.conf") is False:
sleep(2)
+ if USE_CC_NODES:
+ # Wait until we have two node_group since the cluster controller
+ # might take some time to finish its configuration and contact the
+ # region controller.
+ while not len(node_groups) == 2:
+ output, err = self._run_maas_cli(["node-groups", "list"])
+ node_groups = loads(output)
+
+ for ng in node_groups:
+ if ng['uuid'] != region_uuid:
+ cluster_uuid = ng['uuid']
+ break
+ # Configure the cluster controller DHCP server.
+ cluster_dhcp_config = {
+ "ip": "192.168.20.5",
+ "interface": "eth1",
+ "subnet_mask": "255.255.255.0",
+ "broadcast_ip": "192.168.20.255",
+ "router_ip": "192.168.20.1",
+ "management": 2,
+ "ip_range_low": "192.168.20.10",
+ "ip_range_high": "192.168.20.30",
+ }
+ self._set_up_dhcp(cluster_uuid, cluster_dhcp_config)
+
+ # Accept the cluster controller into the MAAS server.
+ output, err = self._run_maas_cli([
+ "node-groups", "accept", "uuid=%s" % cluster_uuid])
+ self.assertThat(
+ output, Contains("Nodegroup(s) accepted."))
def test_10_update_dns_config(self):
#XXX: matsubara Could be asked by maas-dns package and configurable
@@ -334,18 +356,18 @@
dns_config = open("/etc/bind/named.conf.options", 'w')
dns_config.write(LAB_DNS_CONFIG)
dns_config.close()
- output, err = self._run_command(["service", "bind9", "restart"])
+ output, err = run_command(["service", "bind9", "restart"])
self.assertThat(output, Contains(
'* Starting domain name service... bind9'))
def power_on(self, ip, user, password):
- output, err = self._run_command(
+ output, err = run_command(
["ipmipower", "-h", ip, "-u", user, "-p", password, "--on"])
self.addDetail('IPMI power on %s' % ip, text_content(str(output)))
self.assertThat(output, Contains("%s: ok" % ip))
def power_off(self, ip, user, password):
- output, err = self._run_command(
+ output, err = run_command(
["ipmipower", "-h", ip, "-u", user, "-p", password, "--off"])
self.addDetail('IPMI power off %s' % ip, text_content(str(output)))
self.assertThat(output, Contains("%s: ok" % ip))
@@ -360,7 +382,17 @@
self.power_off(ipmi_address, 'admin', 'admin')
self.power_on(ipmi_address, 'admin', 'admin')
+ # XXX: matsubara bug=1108319
+ # revert the timeout back to the original value once bug is fixed.
+ #@timeout(3*60)
+ @timeout(8*60)
def test_11_boot_nodes_enlist(self):
+ if USE_CC_NODES:
+ # XXX: matsubara bug=1108319
+ # The maas-cli doesn't have a way to tell if cluster controllers
+ # are missing the boot image.
+ while not BootImage.objects.all().exists():
+ sleep(10)
self._boot_nodes()
def _wait_nodes(self, status, min_node=None):
@@ -379,7 +411,7 @@
filtered_list = [
node for node in node_list if node['status'] == status]
- @timeout(5*60)
+ @timeout(6*60)
def test_12_check_nodes_declared(self):
self._wait_nodes(0)
@@ -425,7 +457,7 @@
setup_local_dns()
def _run_juju_command(self, args):
- output, err = self._run_command(["juju"] + args)
+ output, err = run_command(["juju"] + args)
# For some reason, in the log, juju replaces '-' by '_'.
command_name = args[0].replace('-', '_')
self.assertIn(
@@ -525,3 +557,18 @@
def test_20_juju_add_unit_mediawiki(self):
self._wait_machines_running(4)
self._wait_units_started('mediawiki', 2)
+
+ @classmethod
+ def tearDownClass(cls):
+ """Signal to the cluster that the region controller tests finished."""
+ if not USE_CC_NODES:
+ raise SkipTest("Not testing Cluster controller")
+ cluster_ssh_command = [
+ "ssh", "-o", "UserKnownHostsFile=/dev/null",
+ "-o", "StrictHostKeyChecking=no",
+ "-o", "CheckHostIP=no",
+ "-i", "/home/ubuntu/adtkey",
+ "-t", "-o", "BatchMode=yes", "-l", "ubuntu",
+ CLUSTER_CONTROLLER_IP]
+ signal_cmd = ["touch", SIGNAL_FILE]
+ run_command(cluster_ssh_command + signal_cmd)
=== modified file 'maas-package-test'
--- maas-package-test 2012-12-06 02:22:18 +0000
+++ maas-package-test 2013-02-14 13:55:23 +0000
@@ -1,4 +1,5 @@
-#!/bin/sh
+#!/bin/bash
set -e -u
exec 2>&1
-nosetests --verbose --stop debian/tests/maas-integration.py
+pip install --no-index --find-links file:///home/ubuntu/ nose-timer
+nosetests --verbose --stop --with-timer debian/tests/maas-integration.py:TestMAASIntegration
=== added file 'utils.py'
--- utils.py 1970-01-01 00:00:00 +0000
+++ utils.py 2013-02-14 13:55:23 +0000
@@ -0,0 +1,73 @@
+# Timeout decorator from
+# http://stackoverflow.com/questions/2281850/timeout-function-if-it-takes-too-long-to-finish
+from subprocess import Popen, PIPE
+from tempfile import mktemp
+from functools import wraps
+import errno
+import os
+import signal
+
+
+SIGNAL_FILE = mktemp("maas-sig-file")
+
+
+class TimeoutError(Exception):
+ pass
+
+
+def timeout(seconds=10, error_message=os.strerror(errno.ETIME)):
+ def decorator(func):
+ def _handle_timeout(signum, frame):
+ raise TimeoutError(error_message)
+
+ def wrapper(*args, **kwargs):
+ signal.signal(signal.SIGALRM, _handle_timeout)
+ signal.alarm(seconds)
+ try:
+ result = func(*args, **kwargs)
+ finally:
+ signal.alarm(0)
+ return result
+
+ return wraps(func)(wrapper)
+
+ return decorator
+
+
+def run_command(args):
+ """A wrapper to Popen to run commands in the command-line."""
+ process = Popen(args, stdout=PIPE, stderr=PIPE, stdin=PIPE)
+ stdout, stderr = process.communicate()
+ return stdout, stderr
+
+
+def update_pxe_config():
+ """Update the files necessary for PXE booting."""
+ ephemerals_fd = open("/etc/maas/import_ephemerals", "r+")
+ ephemerals = ephemerals_fd.read()
+ # Update mirrors for both trunk and the quantal SRU.
+ ephemerals_snippet = (
+ '\n'
+ 'CLOUD_IMAGES_ARCHIVE="http://10.98.0.13/mirrors/maas-ephemeral"\n'
+ 'REMOTE_IMAGES_MIRROR="http://10.98.0.13/mirrors/maas-ephemeral"\n'
+ # XXX: rvb 2013-01-18: Do not use the daily ephemeral images as
+ # they seem to be broken (more precisely, the image from 20130107 is
+ # and the one from 20121008 isn't); investigation is underway.
+ # 'STREAM=daily\n'
+ )
+ ephemerals += ephemerals_snippet
+ ephemerals_fd.seek(0)
+ ephemerals_fd.write(ephemerals)
+ ephemerals_fd.close()
+ # XXX: matsubara Bug=1074167
+ # Update import_pxe_files to not download squashfs images
+ # and use a proxy.
+ pxe_fd = open('/etc/maas/import_pxe_files', "r+")
+ pxe_file = pxe_fd.read()
+ pxe_snippet = 'export http_proxy="http://10.98.0.13:3128"\n'
+ pxe_file = pxe_snippet + pxe_file
+ pxe_fd.seek(0)
+ pxe_fd.write(pxe_file)
+ pxe_fd.close()
+
+
