launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #12619
[Merge] lp:~jtv/maas/demo-celery-config-changes into lp:maas
Jeroen T. Vermeulen has proposed merging lp:~jtv/maas/demo-celery-config-changes into lp:maas.
Commit message:
Miscellaneous fixes related to celeryconfig: read it in the proper way (can't just import from it); find different ways to patch the values in tests; provide a CLUSTER_UUID in the demo setup; and make the celery log file location configurable.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~jtv/maas/demo-celery-config-changes/+merge/126906
Done in collaboration with Raphael.
Jeroen
--
https://code.launchpad.net/~jtv/maas/demo-celery-config-changes/+merge/126906
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~jtv/maas/demo-celery-config-changes into lp:maas.
=== modified file 'etc/celeryconfig.py'
--- etc/celeryconfig.py 2012-09-28 01:07:17 +0000
+++ etc/celeryconfig.py 2012-09-28 10:25:36 +0000
@@ -54,6 +54,8 @@
# local MAAS Celery config.
CLUSTER_UUID = None
+# Location for log file.
+MAAS_CELERY_LOG = '/var/log/maas/celery.log'
WORKER_QUEUE_DNS = 'celery'
WORKER_QUEUE_BOOT_IMAGES = 'celery'
=== modified file 'etc/democeleryconfig.py'
--- etc/democeleryconfig.py 2012-08-01 13:51:26 +0000
+++ etc/democeleryconfig.py 2012-09-28 10:25:36 +0000
@@ -34,3 +34,13 @@
DHCP_CONFIG_FILE = os.path.join(
DEV_ROOT_DIRECTORY, 'run/dhcpd.conf')
+
+DHCP_LEASES_FILE = os.path.join(
+ DEV_ROOT_DIRECTORY, 'run/dhcpd.leases')
+
+
+CLUSTER_UUID = "adfd3977-f251-4f2c-8d61-745dbd690bfc"
+
+
+MAAS_CELERY_LOG = os.path.join(
+ DEV_ROOT_DIRECTORY, 'logs/celeryd/current')
=== modified file 'src/provisioningserver/dhcp/leases.py'
--- src/provisioningserver/dhcp/leases.py 2012-09-10 14:27:00 +0000
+++ src/provisioningserver/dhcp/leases.py 2012-09-28 10:25:36 +0000
@@ -43,7 +43,7 @@
MAASDispatcher,
MAASOAuth,
)
-from celeryconfig import DHCP_LEASES_FILE
+from celery.app import app_or_default
from provisioningserver import cache
from provisioningserver.auth import (
get_recorded_api_credentials,
@@ -61,9 +61,14 @@
LEASES_CACHE_KEY = 'recorded_leases'
+def get_leases_file():
+ """Get the location of the DHCP leases file from the config."""
+ return app_or_default().conf.DHCP_LEASES_FILE
+
+
def get_leases_timestamp():
"""Return the last modification timestamp of the DHCP leases file."""
- return stat(DHCP_LEASES_FILE).st_mtime
+ return stat(get_leases_file()).st_mtime
def parse_leases_file():
@@ -73,7 +78,7 @@
modification time of the leases file, and `leases` is a dict
mapping leased IP addresses to their associated MAC addresses.
"""
- with open(DHCP_LEASES_FILE, 'rb') as leases_file:
+ with open(get_leases_file(), 'rb') as leases_file:
contents = leases_file.read().decode('utf-8')
return fstat(leases_file.fileno()).st_mtime, parse_leases(contents)
=== modified file 'src/provisioningserver/dhcp/tests/test_leases.py'
--- src/provisioningserver/dhcp/tests/test_leases.py 2012-09-10 14:51:47 +0000
+++ src/provisioningserver/dhcp/tests/test_leases.py 2012-09-28 10:25:36 +0000
@@ -72,7 +72,7 @@
def redirect_parser(self, path):
"""Make the leases parser read from a file at `path`."""
- self.patch(leases_module, 'DHCP_LEASES_FILE', path)
+ self.patch(leases_module, 'get_leases_file').return_value = path
def fake_leases_file(self, leases=None, age=None):
"""Fake the presence of a leases file.
=== modified file 'src/provisioningserver/power/poweraction.py'
--- src/provisioningserver/power/poweraction.py 2012-06-28 07:09:01 +0000
+++ src/provisioningserver/power/poweraction.py 2012-09-28 10:25:36 +0000
@@ -20,7 +20,7 @@
import os
import subprocess
-from celeryconfig import POWER_TEMPLATES_DIR
+from celery.app import app_or_default
from provisioningserver.utils import ShellTemplate
@@ -32,6 +32,11 @@
"""Raised when there's a problem executing a power script."""
+def get_power_templates_dir():
+ """Get the power-templates directory from the config."""
+ return app_or_default().conf.POWER_TEMPLATES_DIR
+
+
class PowerAction:
"""Actions for power-related operations.
@@ -53,13 +58,14 @@
@property
def template_basedir(self):
"""Directory where power templates are stored."""
- if POWER_TEMPLATES_DIR is None:
+ power_templates_dir = get_power_templates_dir()
+ if power_templates_dir is None:
# The power templates are installed into the same location
# as this file, and also live in the same directory as this
# file in the source tree.
return os.path.join(os.path.dirname(__file__), 'templates')
else:
- return POWER_TEMPLATES_DIR
+ return power_templates_dir
def get_template(self):
with open(self.path, "rb") as f:
=== modified file 'src/provisioningserver/power/tests/test_poweraction.py'
--- src/provisioningserver/power/tests/test_poweraction.py 2012-09-17 17:27:02 +0000
+++ src/provisioningserver/power/tests/test_poweraction.py 2012-09-28 10:25:36 +0000
@@ -18,6 +18,7 @@
from maastesting.factory import factory
from maastesting.testcase import TestCase
+from mock import Mock
from provisioningserver.enum import POWER_TYPE
import provisioningserver.power.poweraction
from provisioningserver.power.poweraction import (
@@ -38,7 +39,8 @@
def configure_templates_dir(self, path=None):
"""Configure POWER_TEMPLATES_DIR to `path`."""
self.patch(
- provisioningserver.power.poweraction, 'POWER_TEMPLATES_DIR', path)
+ provisioningserver.power.poweraction, 'get_power_templates_dir',
+ Mock(return_value=path))
def test_init_raises_for_unknown_powertype(self):
powertype = factory.make_name("powertype", sep='')
=== modified file 'src/provisioningserver/start_cluster_controller.py'
--- src/provisioningserver/start_cluster_controller.py 2012-09-27 11:35:39 +0000
+++ src/provisioningserver/start_cluster_controller.py 2012-09-28 10:25:36 +0000
@@ -30,7 +30,7 @@
MAASDispatcher,
NoAuth,
)
-from celeryconfig import CLUSTER_UUID
+from celery.app import app_or_default
from provisioningserver.logging import task_logger
from provisioningserver.network import discover_networks
@@ -56,6 +56,16 @@
return MAASClient(NoAuth(), MAASDispatcher(), server_url)
+def get_cluster_uuid():
+ """Read this cluster's UUID from the config."""
+ return app_or_default().conf.CLUSTER_UUID
+
+
+def get_maas_celery_log():
+ """Read location for MAAS Celery log file from the config."""
+ return app_or_default().conf.MAAS_CELERY_LOG
+
+
def register(server_url):
"""Request Rabbit connection details from the domain controller.
@@ -73,10 +83,11 @@
interfaces = json.dumps(discover_networks())
client = make_anonymous_api_client(server_url)
+ cluster_uuid = get_cluster_uuid()
try:
response = client.post(
'api/1.0/nodegroups/', 'register',
- interfaces=interfaces, uuid=CLUSTER_UUID)
+ interfaces=interfaces, uuid=cluster_uuid)
except HTTPError as e:
status_code = e.code
if e.code not in known_responses:
@@ -112,10 +123,10 @@
command = [
'celeryd',
- '--logfile=/var/log/maas/celery.log',
+ '--logfile=%s' % get_maas_celery_log(),
'--loglevel=INFO',
'--beat',
- '-Q', CLUSTER_UUID,
+ '-Q', get_cluster_uuid(),
]
Popen(command, env=env)
=== modified file 'src/provisioningserver/tasks.py'
--- src/provisioningserver/tasks.py 2012-09-25 14:45:59 +0000
+++ src/provisioningserver/tasks.py 2012-09-28 10:25:36 +0000
@@ -28,13 +28,8 @@
check_call,
)
+from celery.app import app_or_default
from celery.task import task
-from celeryconfig import (
- DHCP_CONFIG_FILE,
- DHCP_INTERFACES_FILE,
- WORKER_QUEUE_BOOT_IMAGES,
- WORKER_QUEUE_DNS,
- )
from provisioningserver import boot_images
from provisioningserver.auth import (
record_api_credentials,
@@ -63,6 +58,9 @@
}
+celery_config = app_or_default().conf
+
+
@task
def refresh_secrets(**kwargs):
"""Update the worker's knowledge of various secrets it needs.
@@ -157,7 +155,7 @@
RNDC_COMMAND_RETRY_DELAY = 2
-@task(max_retries=RNDC_COMMAND_MAX_RETRY, queue=WORKER_QUEUE_DNS)
+@task(max_retries=RNDC_COMMAND_MAX_RETRY, queue=celery_config.WORKER_QUEUE_DNS)
def rndc_command(arguments, retry=False, callback=None):
"""Use rndc to execute a command.
:param arguments: Argument list passed down to the rndc command.
@@ -179,7 +177,7 @@
callback.delay()
-@task(queue=WORKER_QUEUE_DNS)
+@task(queue=celery_config.WORKER_QUEUE_DNS)
def write_full_dns_config(zones=None, callback=None, **kwargs):
"""Write out the DNS configuration files: the main configuration
file and the zone files.
@@ -200,7 +198,7 @@
callback.delay()
-@task(queue=WORKER_QUEUE_DNS)
+@task(queue=celery_config.WORKER_QUEUE_DNS)
def write_dns_config(zones=(), callback=None, **kwargs):
"""Write out the DNS configuration file.
@@ -217,7 +215,7 @@
callback.delay()
-@task(queue=WORKER_QUEUE_DNS)
+@task(queue=celery_config.WORKER_QUEUE_DNS)
def write_dns_zone_config(zone, callback=None, **kwargs):
"""Write out a DNS zone configuration file.
@@ -233,7 +231,7 @@
callback.delay()
-@task(queue=WORKER_QUEUE_DNS)
+@task(queue=celery_config.WORKER_QUEUE_DNS)
def setup_rndc_configuration(callback=None):
"""Write out the two rndc configuration files (rndc.conf and
named.conf.rndc).
@@ -315,8 +313,10 @@
DHCP server should listen on.
:param **kwargs: Keyword args passed to dhcp.config.get_config()
"""
- sudo_write_file(DHCP_CONFIG_FILE, config.get_config(**kwargs))
- sudo_write_file(DHCP_INTERFACES_FILE, kwargs.get('dhcp_interfaces', ''))
+ sudo_write_file(
+ celery_config.DHCP_CONFIG_FILE, config.get_config(**kwargs))
+ sudo_write_file(
+ celery_config.DHCP_INTERFACES_FILE, kwargs.get('dhcp_interfaces', ''))
if callback is not None:
callback.delay()
@@ -332,7 +332,7 @@
# =====================================================================
-@task(queue=WORKER_QUEUE_BOOT_IMAGES)
+@task(queue=celery_config.WORKER_QUEUE_BOOT_IMAGES)
def report_boot_images():
"""For master worker only: report available netboot images."""
boot_images.report_to_server()
=== modified file 'src/provisioningserver/tests/test_start_cluster_controller.py'
--- src/provisioningserver/tests/test_start_cluster_controller.py 2012-09-27 09:19:24 +0000
+++ src/provisioningserver/tests/test_start_cluster_controller.py 2012-09-28 10:25:36 +0000
@@ -26,6 +26,7 @@
from apiclient.testing.django import parse_headers_and_body_with_django
from fixtures import EnvironmentVariableFixture
from maastesting.factory import factory
+from mock import Mock
from provisioningserver import start_cluster_controller
from provisioningserver.testing.testcase import PservTestCase
@@ -79,8 +80,9 @@
self.patch(start_cluster_controller, 'sleep').side_effect = Sleeping()
self.patch(start_cluster_controller, 'Popen').side_effect = (
Executing())
- self.patch(
- start_cluster_controller, 'CLUSTER_UUID', factory.getRandomUUID())
+ self.cluster_uuid = factory.getRandomUUID()
+ self.patch(start_cluster_controller, 'get_cluster_uuid', Mock(
+ return_value=self.cluster_uuid))
def make_connection_details(self):
return {
@@ -168,8 +170,6 @@
def test_register_passes_cluster_information(self):
self.prepare_success_response()
- self.patch(
- start_cluster_controller, 'CLUSTER_UUID', factory.getRandomUUID())
interface = {
'interface': factory.make_name('eth'),
'ip': factory.getRandomIPAddress(),
@@ -184,7 +184,7 @@
headers, body = kwargs["headers"], kwargs["data"]
post, files = self.parse_headers_and_body(headers, body)
self.assertEqual([interface], json.loads(post['interfaces']))
- self.assertEqual(start_cluster_controller.CLUSTER_UUID, post['uuid'])
+ self.assertEqual(self.cluster_uuid, post['uuid'])
def test_starts_up_once_accepted(self):
self.patch(start_cluster_controller, 'start_up')
=== modified file 'src/provisioningserver/tests/test_tasks.py'
--- src/provisioningserver/tests/test_tasks.py 2012-09-25 14:45:59 +0000
+++ src/provisioningserver/tests/test_tasks.py 2012-09-28 10:25:36 +0000
@@ -24,12 +24,7 @@
from apiclient.creds import convert_tuple_to_string
from apiclient.maas_client import MAASClient
from apiclient.testing.credentials import make_api_credentials
-from celeryconfig import (
- DHCP_CONFIG_FILE,
- DHCP_INTERFACES_FILE,
- WORKER_QUEUE_BOOT_IMAGES,
- WORKER_QUEUE_DNS,
- )
+from celery.app import app_or_default
from maastesting.celery import CeleryFixture
from maastesting.factory import factory
from maastesting.fakemethod import (
@@ -92,6 +87,9 @@
arbitrary_mac = "AA:BB:CC:DD:EE:FF"
+celery_config = app_or_default().conf
+
+
class TestRefreshSecrets(PservTestCase):
"""Tests for the `refresh_secrets` task."""
@@ -253,7 +251,7 @@
# It should construct Popen with the right parameters.
mocked_popen.assert_any_call(
["sudo", "-n", "maas-provision", "atomic-write", "--filename",
- DHCP_CONFIG_FILE, "--mode", "0644"], stdin=PIPE)
+ celery_config.DHCP_CONFIG_FILE, "--mode", "0644"], stdin=PIPE)
# It should then pass the content to communicate().
content = config.get_config(**config_params).encode("ascii")
@@ -263,7 +261,7 @@
# /var/lib/maas/dhcpd-interfaces.
mocked_popen.assert_any_call(
["sudo", "-n", "maas-provision", "atomic-write", "--filename",
- DHCP_INTERFACES_FILE, "--mode", "0644"], stdin=PIPE)
+ celery_config.DHCP_INTERFACES_FILE, "--mode", "0644"], stdin=PIPE)
def test_restart_dhcp_server_sends_command(self):
recorder = FakeMethod()
@@ -313,7 +311,9 @@
result)
def test_write_dns_config_attached_to_dns_worker_queue(self):
- self.assertEqual(write_dns_config.queue, WORKER_QUEUE_DNS)
+ self.assertEqual(
+ write_dns_config.queue,
+ celery_config.WORKER_QUEUE_DNS)
def test_write_dns_zone_config_writes_file(self):
command = factory.getRandomString()
@@ -344,7 +344,9 @@
result)
def test_write_dns_zone_config_attached_to_dns_worker_queue(self):
- self.assertEqual(write_dns_zone_config.queue, WORKER_QUEUE_DNS)
+ self.assertEqual(
+ write_dns_zone_config.queue,
+ celery_config.WORKER_QUEUE_DNS)
def test_setup_rndc_configuration_writes_files(self):
command = factory.getRandomString()
@@ -369,7 +371,9 @@
result)
def test_setup_rndc_configuration_attached_to_dns_worker_queue(self):
- self.assertEqual(setup_rndc_configuration.queue, WORKER_QUEUE_DNS)
+ self.assertEqual(
+ setup_rndc_configuration.queue,
+ celery_config.WORKER_QUEUE_DNS)
def test_rndc_command_execute_command(self):
command = factory.getRandomString()
@@ -412,7 +416,7 @@
CalledProcessError, rndc_command.delay, command, retry=True)
def test_rndc_command_attached_to_dns_worker_queue(self):
- self.assertEqual(rndc_command.queue, WORKER_QUEUE_DNS)
+ self.assertEqual(rndc_command.queue, celery_config.WORKER_QUEUE_DNS)
def test_write_full_dns_config_sets_up_config(self):
# write_full_dns_config writes the config file, writes
@@ -447,7 +451,9 @@
)))
def test_write_full_dns_attached_to_dns_worker_queue(self):
- self.assertEqual(write_full_dns_config.queue, WORKER_QUEUE_DNS)
+ self.assertEqual(
+ write_full_dns_config.queue,
+ celery_config.WORKER_QUEUE_DNS)
class TestBootImagesTasks(PservTestCase):
@@ -470,4 +476,6 @@
self.assertItemsEqual([image], json.loads(kwargs['images']))
def test_report_boot_images_attached_to_boot_images_worker_queue(self):
- self.assertEqual(write_dns_config.queue, WORKER_QUEUE_BOOT_IMAGES)
+ self.assertEqual(
+ write_dns_config.queue,
+ celery_config.WORKER_QUEUE_BOOT_IMAGES)
