yahoo-eng-team team mailing list archive

Thread
Date

[Bug 1602373] [NEW] cloud-init doesn't always land files that one expects

To: yahoo-eng-team@xxxxxxxxxxxxxxxxxxx
From: Amrith <1602373@xxxxxxxxxxxxxxxxxx>
Date: Tue, 12 Jul 2016 18:01:52 -0000
Reply-to: Bug 1602373 <1602373@xxxxxxxxxxxxxxxxxx>
Sender: bounces@xxxxxxxxxxxxx

Public bug reported:

Trove launches instances using the servers.create() API with some files.
Trove provides a dictionary of files that it wants on the instance and
most of the time this works. Nova passes them to the launched VM as
metadata on config drive.

Sometimes though, it doesn't.

When injection fails, I see a cloud-init.log that looks like this:

https://gist.github.com/amrith/7566d8fef4b6e813cca77e5e3b1f1d5a

When injection succeeds, I see a cloud-init.log that looks like this:

https://gist.github.com/amrith/50d9e3050d88ec51e13b0a510bd718c3

Observe that the one that succeeds properly injects three files:

/etc/injected_files (which is something I added just for debugging)
/etc/trove/conf.d/... two files here ...

On a machine where this injection failed:

root@m10:/tmp/zz/openstack/content# ls -l /etc/trove
total 4
drwxr-xr-x 2 amrith root 4096 Jul 12 16:55 conf.d
root@m10:/tmp/zz/openstack/content# ls -l /etc/trove/conf.d/
total 4
root@m10:/tmp/zz/openstack/content#

Clearly, no files made it over. Yet, the files are definitely there on
the config drive ...

I've mounted the config drive.

root@m10:/tmp/zz/openstack/content# mount | grep zz
/dev/sr0 on /tmp/zz type iso9660 (ro,relatime)

root@m10:/tmp/zz/openstack/content# cd /tmp/zz
root@m10:/tmp/zz# find .
.
./ec2
./ec2/2009-04-04
./ec2/2009-04-04/meta-data.json
./ec2/latest
./ec2/latest/meta-data.json
./openstack
./openstack/2012-08-10
./openstack/2012-08-10/meta_data.json
./openstack/2013-04-04
./openstack/2013-04-04/meta_data.json
./openstack/2013-10-17
./openstack/2013-10-17/meta_data.json
./openstack/2013-10-17/vendor_data.json
./openstack/2015-10-15
./openstack/2015-10-15/meta_data.json
./openstack/2015-10-15/network_data.json
./openstack/2015-10-15/vendor_data.json
./openstack/2016-06-30
./openstack/2016-06-30/meta_data.json
./openstack/2016-06-30/network_data.json
./openstack/2016-06-30/vendor_data.json
./openstack/content
./openstack/content/0000
./openstack/content/0001
./openstack/content/0002
./openstack/latest
./openstack/latest/meta_data.json
./openstack/latest/network_data.json
./openstack/latest/vendor_data.json

root@m10:/tmp/zz# cd openstack/latest/
root@m10:/tmp/zz/openstack/latest# cat meta_data.json
{"files": [{"path": "/etc/injected_files", "content_path": "/content/0000"}, {"path": "/etc/trove/conf.d/guest_info.conf", "content_path": "/content/0001"}, {"path": "/etc/trove/conf.d/trove-guestagent.conf", "content_path": "/content/0002"}], "admin_pass": "pf2ng7tT8JSt", "random_seed": "uSNqQ3udiKspue4y8R8b4gg33Qe6klYYJa8ebvDS0t4i88rq2Owu4yC8TysAfsmsYpno0rbWpWisiTQ3raAOPsx+kGgkrunM9UR5khs/1XRh60tlpKJVIHZnKpLv4PcisLKL2DDoHaaFLV9lPcVtZi3iTqKu6RhcwFyhh+A0mD0xg2G89qACD/RNhWiAuQs4X8le+qgIeoRb3wwg7+4dpujLiqKyx7edDgs9zaEsng21YxO0kv47PiwQf7GoIXObR3xtBYClQQfWQ8a1o35gpDPkPSowqHNKJIxFnIrdFgVHBK5EFTAmwN2JY/GcLwQxLydK02+aalem+bznOpsa1Jl86rNAI9pwltyFYDVz8CGwy46pNYtCOJ2W5WZ+wF4KssE0LkxeWcq7w7CXoFCgz42k13ki0gcHYHw0ieJTMULQT3k65gLwGL2/EsJrZjgil5AaCQhLBLkY2sPhHKYs0k+331QttHHs7PjoX/0as63iNiY62M10givhwyrPDaeYSOAidD0PfRpvVZVcQym/jgKmytV6ZU8DBDwAujGtHjdXK4AovcAvqc96H97arahQr/1SHk8RktyedYjqLC6iYwbSFSHTs+7Fb07MspWKQ2cJLlpmytd/lcZHYAElF/b0VBhR8/eU0dUqdpxjvfGsx8T9/rflMaM5wrDlhad/ikQ=", "uuid": "38731f42-611e-465f-b3e4-1e80aa623caf", "availability_zone": "nova", "hostname": "m10.novalocal", "launch_index": 0, "devices": [], "project_id": "eeec3236ef004821a77ccbd26b81f18f", "name": "m10"}

root@m10:/tmp/zz/openstack/latest#

Sure enough, the files I want are in content/0000, 0001 and 0002

root@m10:/tmp/zz/openstack/content# ls -l
total 4
-r--r--r-- 1 root root 1329 Jul 12 16:55 0000
-r--r--r-- 1 root root  127 Jul 12 16:55 0001
-r--r--r-- 1 root root 1050 Jul 12 16:55 0002

** Affects: cloud-init
     Importance: Undecided
         Status: New

** Attachment added: "this is the cloud-init log file when injection succeeds"
   https://bugs.launchpad.net/bugs/1602373/+attachment/4699588/+files/50d9e3050d88ec51e13b0a510bd718c3-ef60b84e3347f585010397361b62889e146d3cae.zip

-- 
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to OpenStack Compute (nova).
https://bugs.launchpad.net/bugs/1602373

Title:
  cloud-init doesn't always land files that one expects

Status in cloud-init:
  New

Bug description:
  Trove launches instances using the servers.create() API with some
  files. Trove provides a dictionary of files that it wants on the
  instance and most of the time this works. Nova passes them to the
  launched VM as metadata on config drive.

  Sometimes though, it doesn't.

  When injection fails, I see a cloud-init.log that looks like this:

  https://gist.github.com/amrith/7566d8fef4b6e813cca77e5e3b1f1d5a

  When injection succeeds, I see a cloud-init.log that looks like this:

  https://gist.github.com/amrith/50d9e3050d88ec51e13b0a510bd718c3

  Observe that the one that succeeds properly injects three files:

  /etc/injected_files (which is something I added just for debugging)
  /etc/trove/conf.d/... two files here ...

  On a machine where this injection failed:

  root@m10:/tmp/zz/openstack/content# ls -l /etc/trove
  total 4
  drwxr-xr-x 2 amrith root 4096 Jul 12 16:55 conf.d
  root@m10:/tmp/zz/openstack/content# ls -l /etc/trove/conf.d/
  total 4
  root@m10:/tmp/zz/openstack/content#

  Clearly, no files made it over. Yet, the files are definitely there on
  the config drive ...

  I've mounted the config drive.

  root@m10:/tmp/zz/openstack/content# mount | grep zz
  /dev/sr0 on /tmp/zz type iso9660 (ro,relatime)

  root@m10:/tmp/zz/openstack/content# cd /tmp/zz
  root@m10:/tmp/zz# find .
  .
  ./ec2
  ./ec2/2009-04-04
  ./ec2/2009-04-04/meta-data.json
  ./ec2/latest
  ./ec2/latest/meta-data.json
  ./openstack
  ./openstack/2012-08-10
  ./openstack/2012-08-10/meta_data.json
  ./openstack/2013-04-04
  ./openstack/2013-04-04/meta_data.json
  ./openstack/2013-10-17
  ./openstack/2013-10-17/meta_data.json
  ./openstack/2013-10-17/vendor_data.json
  ./openstack/2015-10-15
  ./openstack/2015-10-15/meta_data.json
  ./openstack/2015-10-15/network_data.json
  ./openstack/2015-10-15/vendor_data.json
  ./openstack/2016-06-30
  ./openstack/2016-06-30/meta_data.json
  ./openstack/2016-06-30/network_data.json
  ./openstack/2016-06-30/vendor_data.json
  ./openstack/content
  ./openstack/content/0000
  ./openstack/content/0001
  ./openstack/content/0002
  ./openstack/latest
  ./openstack/latest/meta_data.json
  ./openstack/latest/network_data.json
  ./openstack/latest/vendor_data.json

  root@m10:/tmp/zz# cd openstack/latest/
  root@m10:/tmp/zz/openstack/latest# cat meta_data.json
  {"files": [{"path": "/etc/injected_files", "content_path": "/content/0000"}, {"path": "/etc/trove/conf.d/guest_info.conf", "content_path": "/content/0001"}, {"path": "/etc/trove/conf.d/trove-guestagent.conf", "content_path": "/content/0002"}], "admin_pass": "pf2ng7tT8JSt", "random_seed": "uSNqQ3udiKspue4y8R8b4gg33Qe6klYYJa8ebvDS0t4i88rq2Owu4yC8TysAfsmsYpno0rbWpWisiTQ3raAOPsx+kGgkrunM9UR5khs/1XRh60tlpKJVIHZnKpLv4PcisLKL2DDoHaaFLV9lPcVtZi3iTqKu6RhcwFyhh+A0mD0xg2G89qACD/RNhWiAuQs4X8le+qgIeoRb3wwg7+4dpujLiqKyx7edDgs9zaEsng21YxO0kv47PiwQf7GoIXObR3xtBYClQQfWQ8a1o35gpDPkPSowqHNKJIxFnIrdFgVHBK5EFTAmwN2JY/GcLwQxLydK02+aalem+bznOpsa1Jl86rNAI9pwltyFYDVz8CGwy46pNYtCOJ2W5WZ+wF4KssE0LkxeWcq7w7CXoFCgz42k13ki0gcHYHw0ieJTMULQT3k65gLwGL2/EsJrZjgil5AaCQhLBLkY2sPhHKYs0k+331QttHHs7PjoX/0as63iNiY62M10givhwyrPDaeYSOAidD0PfRpvVZVcQym/jgKmytV6ZU8DBDwAujGtHjdXK4AovcAvqc96H97arahQr/1SHk8RktyedYjqLC6iYwbSFSHTs+7Fb07MspWKQ2cJLlpmytd/lcZHYAElF/b0VBhR8/eU0dUqdpxjvfGsx8T9/rflMaM5wrDlhad/ikQ=", "uuid": "38731f42-611e-465f-b3e4-1e80aa623caf", "availability_zone": "nova", "hostname": "m10.novalocal", "launch_index": 0, "devices": [], "project_id": "eeec3236ef004821a77ccbd26b81f18f", "name": "m10"}

  root@m10:/tmp/zz/openstack/latest#

  Sure enough, the files I want are in content/0000, 0001 and 0002

  root@m10:/tmp/zz/openstack/content# ls -l
  total 4
  -r--r--r-- 1 root root 1329 Jul 12 16:55 0000
  -r--r--r-- 1 root root  127 Jul 12 16:55 0001
  -r--r--r-- 1 root root 1050 Jul 12 16:55 0002

To manage notifications about this bug go to:
https://bugs.launchpad.net/cloud-init/+bug/1602373/+subscriptions

Follow ups

[Bug 1602373] Re: cloud-init doesn't always land files that one expects
From: Scott Moser, 2016-08-10
[Bug 1602373] Re: cloud-init doesn't always land files that one expects
From: Launchpad Bug Tracker, 2016-07-21
[Bug 1602373] Re: cloud-init doesn't always land files that one expects
From: Launchpad Bug Tracker, 2016-07-14
[Bug 1602373] Re: cloud-init doesn't always land files that one expects
From: Scott Moser, 2016-07-13
[Bug 1602373] Re: nova cloud-init doesn't always land files that one expects
From: Amrith, 2016-07-12
[Bug 1602373] Re: nova cloud-init doesn't always land files that one expects
From: Matt Riedemann, 2016-07-12