cloud-init-dev team mailing list archive

[Scott Moser <ssmoser2+ubuntu@xxxxxxxxx>]

Scott Moser has proposed merging ~smoser/cloud-init:fix/templater-with-non-ascii into cloud-init:master.

Commit message:
renderer: support unicode in render_from_file.

If a file passed to render_from_file had non-ascii text then
jinja in python2 would decode as ascii, which would cause
UnicodeDecodeError.  This issue can be re-created in python2
with just:
 'can\xe2\x80\x99t'.decode()

The solution here is to explicitly pass in unicode supporting
type (py3 str, py2 unicode).  Those are six.text_type.
Then jinja does not try to decode.

The reason we hit this is that load_file calls decode_binary.
decode_binary believes it has no work to do if it got a six.string_types.
  isinstance('can\xe2\x80\x99t', six.string_types) == True
So it returns the original string which will blow up for jinja.

Our fix here then is to load the file in binary mode and explicitly
decode it to utf-8.  Then in python2 we'll have a unicode type
and in python3 we'll have a string type.

Requested reviews:
  cloud-init commiters (cloud-init-dev)

For more details, see:
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/343120
-- 
Your team cloud-init commiters is requested to review the proposed merge of ~smoser/cloud-init:fix/templater-with-non-ascii into cloud-init:master.

diff --git a/cloudinit/templater.py b/cloudinit/templater.py
index b3ea64e..17e8f50 100644
--- a/cloudinit/templater.py
+++ b/cloudinit/templater.py
@@ -121,7 +121,8 @@ def detect_template(text):
 def render_from_file(fn, params):
     if not params:
         params = {}
-    template_type, renderer, content = detect_template(util.load_file(fn))
+    template_type, renderer, content = detect_template(
+        util.load_file(fn, decode=False).decode('utf-8'))
     LOG.debug("Rendering content of '%s' using renderer %s", fn, template_type)
     return renderer(content, params)
 
@@ -132,11 +133,15 @@ def render_to_file(fn, outfn, params, mode=0o644):
 
 
 def render_string_to_file(content, outfn, params, mode=0o644):
+    """Render string (or py2 unicode) to file.
+    Warning: py2 str with non-ascii chars will cause UnicodeDecodeError."""
     contents = render_string(content, params)
     util.write_file(outfn, contents, mode=mode)
 
 
 def render_string(content, params):
+    """Render string (or py2 unicode).
+    Warning: py2 str with non-ascii chars will cause UnicodeDecodeError."""
     if not params:
         params = {}
     template_type, renderer, content = detect_template(content)
diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py
index 53154d3..1080e13 100644
--- a/tests/unittests/test_templating.py
+++ b/tests/unittests/test_templating.py
@@ -10,6 +10,7 @@ from cloudinit.tests import helpers as test_helpers
 import textwrap
 
 from cloudinit import templater
+from cloudinit.util import load_file, write_file
 
 try:
     import Cheetah
@@ -19,7 +20,17 @@ except ImportError:
     HAS_CHEETAH = False
 
 
-class TestTemplates(test_helpers.TestCase):
+class TestTemplates(test_helpers.CiTestCase):
+    jinja_utf8 = b'It\xe2\x80\x99s not ascii, {{name}}\n'
+    jinja_utf8_rbob = b'It\xe2\x80\x99s not ascii, bob\n'.decode('utf-8')
+
+    @staticmethod
+    def add_header(renderer, data):
+        """Return text (py2 unicode/py3 str) with template header."""
+        if isinstance(data, bytes):
+            data = data.decode('utf-8')
+        return "## template: %s\n" % renderer + data
+
     def test_render_basic(self):
         in_data = textwrap.dedent("""
             ${b}
@@ -106,4 +117,32 @@ $a,$b'''
                                            'codename': codename})
         self.assertEqual(ex_data, out_data)
 
+    def test_jinja_nonascii_render_to_string(self):
+        """Test jinja render_to_string with non-ascii content."""
+        self.assertEqual(
+            templater.render_string(
+                self.add_header("jinja", self.jinja_utf8), {"name": "bob"}),
+            self.jinja_utf8_rbob)
+
+    def test_jinja_nonascii_render_to_file(self):
+        """Test jinja render_to_file of a filename with non-ascii content."""
+        tmpl_fn = self.tmp_path("j-render-to-file.template")
+        out_fn = self.tmp_path("j-render-to-file.out")
+        write_file(filename=tmpl_fn, omode="wb",
+                   content=self.add_header(
+                       "jinja", self.jinja_utf8).encode('utf-8'))
+        templater.render_to_file(tmpl_fn, out_fn, {"name": "bob"})
+        result = load_file(out_fn, decode=False).decode('utf-8')
+        self.assertEqual(result, self.jinja_utf8_rbob)
+
+    def test_jinja_nonascii_render_from_file(self):
+        """Test jinja render_from_file with non-ascii content."""
+        tmpl_fn = self.tmp_path("j-render-from-file.template")
+        write_file(tmpl_fn, omode="wb",
+                   content=self.add_header(
+                       "jinja", self.jinja_utf8).encode('utf-8'))
+        result = templater.render_from_file(tmpl_fn, {"name": "bob"})
+        self.assertEqual(result, self.jinja_utf8_rbob)
+
+
 # vi: ts=4 expandtab