launchpad-reviewers team mailing list archive

[Raphaël Badin <raphael.badin@xxxxxxxxxxxxx>]

Raphaël Badin has proposed merging lp:~rvb/maas/preseed-customisation into lp:maas.

Requested reviews:
  Launchpad code reviewers (launchpad-reviewers)

For more details, see:
https://code.launchpad.net/~rvb/maas/preseed-customisation/+merge/110861

This branch adds the method 'load_preseed_template' to the preseed module.  This method loads a preseed template form a given node.

= Pre-imp =

This was discussed with Gavin (see http://bit.ly/KXW5ZU for details).  More than that, this branch was actually started by Gavin.  I needed this to continue the preseed work so I decided to finish it off since Gavin is away today and tomorrow.

= Notes =

- Add PRESEED_TEMPLATE_LOCATIONS which lists locations where the templates can be found.

- Change get_preseed_filenames to be able to specify if the default template ('generic') should be returned as a last resort or not: we want that template to be used only when looking for preseed templates, no when looking for parent templates (from which preseed templates inherits).
-- 
https://code.launchpad.net/~rvb/maas/preseed-customisation/+merge/110861
Your team Launchpad code reviewers is requested to review the proposed merge of lp:~rvb/maas/preseed-customisation into lp:maas.

=== modified file 'src/maas/development.py'
--- src/maas/development.py	2012-06-01 06:52:08 +0000
+++ src/maas/development.py	2012-06-18 16:59:26 +0000
@@ -83,5 +83,11 @@
 # on a production MAAS.
 ALLOW_ANONYMOUS_METADATA_ACCESS = True
 
+# Use in-branch preseed templates.
+PRESEED_TEMPLATE_LOCATIONS = (
+    abspath("etc/preseeds"),
+    abspath("contrib/preseeds"),
+    )
+
 # Allow the user to override settings in maas_local_settings.
 import_local_settings()

=== modified file 'src/maas/settings.py'
--- src/maas/settings.py	2012-06-13 11:02:58 +0000
+++ src/maas/settings.py	2012-06-18 16:59:26 +0000
@@ -293,5 +293,12 @@
 # for all nodes, will be exposed on your network.
 ALLOW_ANONYMOUS_METADATA_ACCESS = False
 
+# Earlier locations in the following list will shadow, or overlay, later
+# locations.
+PRESEED_TEMPLATE_LOCATIONS = (
+    "/etc/maas/preseeds",
+    "/usr/share/maas/preseeds",
+    )
+
 # Allow the user to override settings in maas_local_settings.
 import_local_settings()

=== modified file 'src/maasserver/preseed.py'
--- src/maasserver/preseed.py	2012-06-14 16:17:27 +0000
+++ src/maasserver/preseed.py	2012-06-18 16:59:26 +0000
@@ -1,7 +1,7 @@
 # Copyright 2012 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
-"""Preseed module."""
+"""Preseed generation."""
 
 from __future__ import (
     absolute_import,
@@ -12,36 +12,47 @@
 __metaclass__ = type
 __all__ = []
 
+import base64
+from os.path import join
+
+from django.conf import settings
+import tempita
+
 
 GENERIC_FILENAME = 'generic'
 
 
 # XXX: rvb 2012-06-14 bug=1013146:  'precise' is hardcoded here.
-def get_preseed_filenames(node, type, release='precise'):
+def get_preseed_filenames(node, prefix, release='precise', default=False):
     """List possible preseed template filenames for the given node.
 
     :param node: The node to return template preseed filenames for.
     :type node: :class:`maasserver.models.Node`
-    :param type: The preseed type (will be used as a prefix in the template
-        filenames).  Usually one of {'', 'enlist', 'commissioning'}.
-    :type type: basestring
+    :param prefix: At the top level, this is the preseed type (will be used as
+        a prefix in the template filenames).  Usually one of {'', 'enlist',
+        'commissioning'}.
+    :type prefix: basestring
     :param release: The Ubuntu release to be used.
-    :type type: basestring
+    :type release: basestring
+    :param default: Should we return the default ('generic') template as a
+        last resort template?
+    :type default: boolean
 
     Returns a list of possible preseed template filenames using the following
     lookup order:
-    {type}_{node_architecture}_{node_subarchitecture}_{release}_{node_hostname}
-    {type}_{node_architecture}_{node_subarchitecture}_{release}
-    {type}_{node_architecture}
-    {type}
+    {prefix}_{node_architecture}_{node_subarchitecture}_{release}_{node_name}
+    {prefix}_{node_architecture}_{node_subarchitecture}_{release}
+    {prefix}_{node_architecture}
+    {prefix}
     'generic'
     """
     arch = split_subarch(node.architecture)
-    elements = [type] + arch + [release, node.hostname]
+    elements = [prefix] + arch + [release, node.hostname]
     while elements:
         yield compose_filename(elements)
         elements.pop()
-    yield GENERIC_FILENAME
+    if default:
+        yield GENERIC_FILENAME
 
 
 def split_subarch(architecture):
@@ -52,3 +63,59 @@
 def compose_filename(elements):
     """Create a preseed filename from a list of elements."""
     return '_'.join(elements)
+
+
+def get_preseed_template(filenames):
+    """Get the path and content for the first template found.
+
+    :param filenames: An iterable of relative filenames.
+    """
+    assert not isinstance(filenames, basestring)
+    for location in settings.PRESEED_TEMPLATE_LOCATIONS:
+        for filename in filenames:
+            filepath = join(location, filename)
+            try:
+                with open(filepath, "rb") as stream:
+                    content = stream.read()
+                    return filepath, content
+            except IOError:
+                pass  # Ignore.
+    else:
+        return None, None
+
+
+class PreseedTemplate(tempita.Template):
+    """A Tempita template specialised for preseed rendering."""
+
+    default_namespace = dict(
+        tempita.Template.default_namespace,
+        b64decode=base64.b64decode,
+        b64encode=base64.b64encode)
+
+
+class TemplateNotFoundError(Exception):
+    """The template has not been found."""
+
+    def __init__(self, name):
+        super(TemplateNotFoundError, self).__init__(name)
+        self.name = name
+
+
+# XXX: rvb 2012-06-18 bug=1013146:  'precise' is hardcoded here.
+def load_preseed_template(node, prefix, release="precise"):
+    """Find and load a `PreseedTemplate` for the given node.
+
+    :param node: See `get_preseed_filenames`.
+    :param prefix: See `get_preseed_filenames`.
+    :param release: See `get_preseed_filenames`.
+    """
+
+    def get_template(name, from_template, default=False):
+        filenames = get_preseed_filenames(node, name, release, default)
+        filepath, content = get_preseed_template(filenames)
+        if filepath is None:
+            raise TemplateNotFoundError(name)
+        return PreseedTemplate(
+            content, name=filepath, get_template=get_template)
+
+    return get_template(prefix, None, default=True)

=== modified file 'src/maasserver/tests/test_preseed.py'
--- src/maasserver/tests/test_preseed.py	2012-06-15 07:08:42 +0000
+++ src/maasserver/tests/test_preseed.py	2012-06-18 16:59:26 +0000
@@ -1,7 +1,7 @@
 # Copyright 2012 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
-"""Test preseed module."""
+"""Test `maasserver.preseed` and related bits and bobs."""
 
 from __future__ import (
     absolute_import,
@@ -12,15 +12,29 @@
 __metaclass__ = type
 __all__ = []
 
+import base64
+import os
+
+from django.conf import settings
 from maasserver.preseed import (
+    GENERIC_FILENAME,
     get_preseed_filenames,
+    get_preseed_template,
+    load_preseed_template,
+    PreseedTemplate,
     split_subarch,
+    TemplateNotFoundError,
     )
 from maasserver.testing.factory import factory
 from maasserver.testing.testcase import TestCase
-
-
-class TestPreseedUtilities(TestCase):
+from testtools.matchers import (
+    AllMatch,
+    IsInstance,
+    )
+
+
+class TestSplitSubArch(TestCase):
+    """Tests for `split_subarch`."""
 
     def test_split_subarch_returns_list(self):
         self.assertEqual(['amd64'], split_subarch('amd64'))
@@ -28,6 +42,10 @@
     def test_split_subarch_splits_sub_architecture(self):
         self.assertEqual(['amd64', 'test'], split_subarch('amd64/test'))
 
+
+class TestGetPreseedFilenames(TestCase):
+    """Tests for `get_preseed_filenames`."""
+
     def test_get_preseed_filenames_returns_filenames(self):
         hostname = factory.getRandomString()
         type = factory.getRandomString()
@@ -41,7 +59,7 @@
                 '%s' % type,
                 'generic',
             ],
-            list(get_preseed_filenames(node, type, release)))
+            list(get_preseed_filenames(node, type, release, True)))
 
     def test_get_preseed_filenames_returns_filenames_with_subarch(self):
         arch = factory.getRandomString()
@@ -63,4 +81,208 @@
                 '%s' % type,
                 'generic',
             ],
-            list(get_preseed_filenames(node, type, release)))
+            list(get_preseed_filenames(node, type, release, True)))
+
+    def test_get_preseed_filenames_returns_list_without_default(self):
+        # If default=False is passed to get_preseed_filenames, the
+        # returned list won't include the default template name as a
+        # last resort template.
+        hostname = factory.getRandomString()
+        prefix = factory.getRandomString()
+        release = factory.getRandomString()
+        node = factory.make_node(hostname=hostname)
+        self.assertSequenceEqual(
+            'generic',
+            list(get_preseed_filenames(node, prefix, release, True))[-1])
+
+    def test_get_preseed_filenames_returns_list_with_default(self):
+        # If default=True is passed to get_preseed_filenames, the
+        # returned list will include the default template name as a
+        # last resort template.
+        hostname = factory.getRandomString()
+        prefix = factory.getRandomString()
+        release = factory.getRandomString()
+        node = factory.make_node(hostname=hostname)
+        self.assertSequenceEqual(
+            prefix,
+            list(get_preseed_filenames(node, prefix, release, False))[-1])
+
+
+class TestConfiguration(TestCase):
+    """Test for correct configuration of the preseed component."""
+
+    def test_setting_defined(self):
+        self.assertThat(
+            settings.PRESEED_TEMPLATE_LOCATIONS,
+            AllMatch(IsInstance(basestring)))
+
+
+class TestPreseedTemplate(TestCase):
+    """Tests for :class:`PreseedTemplate`."""
+
+    def test_preseed_template_b64decode(self):
+        content = factory.getRandomString()
+        encoded_content = base64.b64encode(content)
+        template = PreseedTemplate("{{b64decode('%s')}}" % encoded_content)
+        self.assertEqual(content, template.substitute())
+
+    def test_preseed_template_b64encode(self):
+        content = factory.getRandomString()
+        template = PreseedTemplate("{{b64encode('%s')}}" % content)
+        self.assertEqual(base64.b64encode(content), template.substitute())
+
+
+class TestGetPreseedTemplate(TestCase):
+    """Tests for `get_preseed_template`."""
+
+    def test_get_preseed_template_returns_None_if_no_template_locations(self):
+        # get_preseed_template() returns None when no template locations are
+        # defined.
+        self.patch(settings, "PRESEED_TEMPLATE_LOCATIONS", [])
+        self.assertEqual(
+            (None, None),
+            get_preseed_template(
+                (factory.getRandomString(), factory.getRandomString())))
+
+    def test_get_preseed_template_returns_None_when_no_filenames(self):
+        # get_preseed_template() returns None when no filenames are passed in.
+        location = self.make_dir()
+        self.patch(settings, "PRESEED_TEMPLATE_LOCATIONS", [location])
+        self.assertEqual((None, None), get_preseed_template(()))
+
+    def test_get_preseed_template_find_template_in_first_location(self):
+        template_content = factory.getRandomString()
+        template_path = self.make_file(contents=template_content)
+        template_filename = os.path.basename(template_path)
+        locations = [
+            os.path.dirname(template_path),
+            self.make_dir(),
+            ]
+        self.patch(settings, "PRESEED_TEMPLATE_LOCATIONS", locations)
+        self.assertEqual(
+            (template_path, template_content),
+            get_preseed_template([template_filename]))
+
+    def test_get_preseed_template_find_template_in_last_location(self):
+        template_content = factory.getRandomString()
+        template_path = self.make_file(contents=template_content)
+        template_filename = os.path.basename(template_path)
+        locations = [
+            self.make_dir(),
+            os.path.dirname(template_path),
+            ]
+        self.patch(settings, "PRESEED_TEMPLATE_LOCATIONS", locations)
+        self.assertEqual(
+            (template_path, template_content),
+            get_preseed_template([template_filename]))
+
+
+class TestLoadPreseedTemplate(TestCase):
+    """Tests for `load_preseed_template`."""
+
+    def create_template(self, location, name, content=None):
+        # Create a tempita template in the given `location` with the
+        # given `name`.  If content is not provided, a random content
+        # will be put inside the template.
+        path = os.path.join(location, name)
+        rendered_content = None
+        if content is None:
+            rendered_content = factory.getRandomString()
+            content = b'{{def stuff}}%s{{enddef}}{{stuff}}' % rendered_content
+        with open(path, "wb") as outf:
+            outf.write(content)
+        return rendered_content
+
+    def test_load_preseed_template_returns_PreseedTemplate(self):
+        location = self.make_dir()
+        self.patch(settings, "PRESEED_TEMPLATE_LOCATIONS", [location])
+        name = factory.getRandomString()
+        self.create_template(location, name)
+        node = factory.make_node()
+        template = load_preseed_template(node, name)
+        self.assertIsInstance(template, PreseedTemplate)
+
+    def test_load_preseed_template_raises_if_no_template(self):
+        location = self.make_dir()
+        self.patch(settings, "PRESEED_TEMPLATE_LOCATIONS", [location])
+        node = factory.make_node()
+        unknown_template_name = factory.getRandomString()
+        self.assertRaises(
+            TemplateNotFoundError, load_preseed_template, node,
+            unknown_template_name)
+
+    def test_load_preseed_template_generic_lookup(self):
+        # The template lookup method ends up picking up a template named
+        # 'generic' if no more specific template exist.
+        location = self.make_dir()
+        self.patch(settings, "PRESEED_TEMPLATE_LOCATIONS", [location])
+        content = self.create_template(location, GENERIC_FILENAME)
+        node = factory.make_node(hostname=factory.getRandomString())
+        template = load_preseed_template(node, factory.getRandomString())
+        self.assertEqual(content, template.substitute())
+
+    def test_load_preseed_template_prefix_lookup(self):
+        # 2nd last in the hierarchy is a template named 'prefix'.
+        location = self.make_dir()
+        prefix = factory.getRandomString()
+        self.patch(settings, "PRESEED_TEMPLATE_LOCATIONS", [location])
+        # Create the generic template.  This one will be ignored due to the
+        # presence of a more specific template.
+        self.create_template(location, GENERIC_FILENAME)
+        # Create the 'prefix' template.  This is the one which will be
+        # picked up.
+        content = self.create_template(location, prefix)
+        node = factory.make_node(hostname=factory.getRandomString())
+        template = load_preseed_template(node, prefix)
+        self.assertEqual(content, template.substitute())
+
+    def test_load_preseed_template_node_specific_lookup(self):
+        # At the top of the lookup hierarchy is a template specific to this
+        # node.  It will be used first if it's present.
+        location = self.make_dir()
+        prefix = factory.getRandomString()
+        release = factory.getRandomString()
+        self.patch(settings, "PRESEED_TEMPLATE_LOCATIONS", [location])
+        # Create the generic and 'prefix' templates.  They will be ignored
+        # due to the presence of a more specific template.
+        self.create_template(location, GENERIC_FILENAME)
+        self.create_template(location, prefix)
+        node = factory.make_node(hostname=factory.getRandomString())
+        node_template_name = "%s_%s_%s_%s" % (
+            prefix, node.architecture, release, node.hostname)
+        # Create the node-specific template.
+        content = self.create_template(location, node_template_name)
+        template = load_preseed_template(node, prefix, release)
+        self.assertEqual(content, template.substitute())
+
+    def test_load_preseed_template_with_inherits(self):
+        # A preseed file can "inherit" from another file.
+        location = self.make_dir()
+        self.patch(settings, "PRESEED_TEMPLATE_LOCATIONS", [location])
+        prefix = factory.getRandomString()
+        # Create preseed template.
+        master_template_name = factory.getRandomString()
+        preseed_content = '{{inherit "%s"}}' % master_template_name
+        self.create_template(location, prefix, preseed_content)
+        master_content = self.create_template(location, master_template_name)
+        node = factory.make_node()
+        template = load_preseed_template(node, prefix)
+        self.assertEqual(master_content, template.substitute())
+
+    def test_load_preseed_template_parent_lookup_doesnt_include_default(self):
+        # The lookup for parent templates does not include the default
+        # 'generic' file.
+        location = self.make_dir()
+        self.patch(settings, "PRESEED_TEMPLATE_LOCATIONS", [location])
+        prefix = factory.getRandomString()
+        # Create 'generic' template.  It won't be used because the
+        # lookup for parent templates does not use the 'generic' template.
+        self.create_template(location, GENERIC_FILENAME)
+        unknown_master_template_name = factory.getRandomString()
+        # Create preseed template.
+        preseed_content = '{{inherit "%s"}}' % unknown_master_template_name
+        self.create_template(location, prefix, preseed_content)
+        node = factory.make_node()
+        template = load_preseed_template(node, prefix)
+        self.assertRaises(
+            TemplateNotFoundError, template.substitute)