openstack team mailing list archive

[Greg C <agregc@xxxxxxxxx>]

I suspended instances and rebooted one of my compute nodes, but not compute
wont start up.  Appears to be the checks of the machine images that fails.
Libvirt has found the instances and started them back up, and compute
checks them out and gets through 4 of 6 instances, then bombs:

>From nova-compute.log:

2012-12-18 12:37:55 DEBUG nova.utils [-] Got semaphore "compute_resources"
for method "update_available_resource"... from (pid=9657) inner
/usr/lib/python2.7/dist-packages/nova/utils.py:713
2012-12-18 12:38:13 DEBUG nova.utils [-] Running cmd (subprocess): env
LC_ALL=C LANG=C qemu-img info
/var/lib/nova/instances/instance-0000003c/disk from (pid=9657) execute
/usr/lib/python2.7/dist-packages/nova/utils.py:176
2012-12-18 12:38:13 DEBUG nova.utils [-] Result was 0 from (pid=9657)
execute /usr/lib/python2.7/dist-packages/nova/utils.py:191
2012-12-18 12:38:13 DEBUG nova.utils [-] Running cmd (subprocess): env
LC_ALL=C LANG=C qemu-img info
/var/lib/nova/instances/instance-0000003c/disk from (pid=9657) execute
/usr/lib/python2.7/dist-packages/nova/utils.py:176
2012-12-18 12:38:13 DEBUG nova.utils [-] Result was 0 from (pid=9657)
execute /usr/lib/python2.7/dist-packages/nova/utils.py:191
2012-12-18 12:38:14 DEBUG nova.utils [-] Running cmd (subprocess): env
LC_ALL=C LANG=C qemu-img info
/var/lib/nova/instances/instance-0000005a/disk from (pid=9657) execute
/usr/lib/python2.7/dist-packages/nova/utils.py:176
2012-12-18 12:38:14 DEBUG nova.utils [-] Result was 0 from (pid=9657)
execute /usr/lib/python2.7/dist-packages/nova/utils.py:191
2012-12-18 12:38:14 DEBUG nova.utils [-] Running cmd (subprocess): env
LC_ALL=C LANG=C qemu-img info
/var/lib/nova/instances/instance-0000005a/disk from (pid=9657) execute
/usr/lib/python2.7/dist-packages/nova/utils.py:176
2012-12-18 12:38:14 DEBUG nova.utils [-] Result was 0 from (pid=9657)
execute /usr/lib/python2.7/dist-packages/nova/utils.py:191
2012-12-18 12:38:14 DEBUG nova.utils [-] Running cmd (subprocess): env
LC_ALL=C LANG=C qemu-img info
/var/lib/nova/instances/instance-0000005a/disk.local from (pid=9657)
execute /usr/lib/python2.7/dist-packages/nova/utils.py:176
2012-12-18 12:38:14 DEBUG nova.utils [-] Result was 0 from (pid=9657)
execute /usr/lib/python2.7/dist-packages/nova/utils.py:191
2012-12-18 12:38:14 DEBUG nova.utils [-] Running cmd (subprocess): env
LC_ALL=C LANG=C qemu-img info
/var/lib/nova/instances/instance-0000005a/disk.local from (pid=9657)
execute /usr/lib/python2.7/dist-packages/nova/utils.py:176
2012-12-18 12:38:14 DEBUG nova.utils [-] Result was 0 from (pid=9657)
execute /usr/lib/python2.7/dist-packages/nova/utils.py:191
2012-12-18 12:38:14 DEBUG nova.utils [-] Running cmd (subprocess): env
LC_ALL=C LANG=C qemu-img info
/var/lib/nova/instances/instance-00000043/disk from (pid=9657) execute
/usr/lib/python2.7/dist-packages/nova/utils.py:176
2012-12-18 12:38:14 DEBUG nova.utils [-] Result was 0 from (pid=9657)
execute /usr/lib/python2.7/dist-packages/nova/utils.py:191
2012-12-18 12:38:14 DEBUG nova.utils [-] Running cmd (subprocess): env
LC_ALL=C LANG=C qemu-img info
/var/lib/nova/instances/instance-00000043/disk from (pid=9657) execute
/usr/lib/python2.7/dist-packages/nova/utils.py:176
2012-12-18 12:38:14 DEBUG nova.utils [-] Result was 0 from (pid=9657)
execute /usr/lib/python2.7/dist-packages/nova/utils.py:191
2012-12-18 12:38:15 DEBUG nova.utils [-] Running cmd (subprocess): env
LC_ALL=C LANG=C qemu-img info
/var/lib/nova/instances/instance-00000058/disk from (pid=9657) execute
/usr/lib/python2.7/dist-packages/nova/utils.py:176
2012-12-18 12:38:15 DEBUG nova.utils [-] Result was 0 from (pid=9657)
execute /usr/lib/python2.7/dist-packages/nova/utils.py:191
2012-12-18 12:38:15 DEBUG nova.utils [-] Running cmd (subprocess): env
LC_ALL=C LANG=C qemu-img info
/var/lib/nova/instances/instance-00000058/disk from (pid=9657) execute
/usr/lib/python2.7/dist-packages/nova/utils.py:176
2012-12-18 12:38:15 DEBUG nova.utils [-] Result was 0 from (pid=9657)
execute /usr/lib/python2.7/dist-packages/nova/utils.py:191
2012-12-18 12:38:15 DEBUG nova.utils [-] Running cmd (subprocess): env
LC_ALL=C LANG=C qemu-img info
/var/lib/nova/instances/instance-00000058/disk.local from (pid=9657)
execute /usr/lib/python2.7/dist-packages/nova/utils.py:176
2012-12-18 12:38:15 DEBUG nova.utils [-] Result was 0 from (pid=9657)
execute /usr/lib/python2.7/dist-packages/nova/utils.py:191
2012-12-18 12:38:15 DEBUG nova.utils [-] Running cmd (subprocess): env
LC_ALL=C LANG=C qemu-img info
/var/lib/nova/instances/instance-00000058/disk.local from (pid=9657)
execute /usr/lib/python2.7/dist-packages/nova/utils.py:176
2012-12-18 12:38:15 DEBUG nova.utils [-] Result was 0 from (pid=9657)
execute /usr/lib/python2.7/dist-packages/nova/utils.py:191
2012-12-18 12:38:16 DEBUG nova.utils [-] Running cmd (subprocess): env
LC_ALL=C LANG=C qemu-img info
/var/lib/nova/instances/instance-0000005e/disk from (pid=9657) execute
/usr/lib/python2.7/dist-packages/nova/utils.py:176
2012-12-18 12:38:16 DEBUG nova.utils [-] Result was 0 from (pid=9657)
execute /usr/lib/python2.7/dist-packages/nova/utils.py:191
2012-12-18 12:38:16 CRITICAL nova [-] string index out of range
2012-12-18 12:38:16 TRACE nova Traceback (most recent call last):
2012-12-18 12:38:16 TRACE nova   File "/usr/bin/nova-compute", line 48, in
<module>
2012-12-18 12:38:16 TRACE nova     service.wait()
2012-12-18 12:38:16 TRACE nova   File
"/usr/lib/python2.7/dist-packages/nova/service.py", line 659, in wait
2012-12-18 12:38:16 TRACE nova     _launcher.wait()
2012-12-18 12:38:16 TRACE nova   File
"/usr/lib/python2.7/dist-packages/nova/service.py", line 192, in wait
2012-12-18 12:38:16 TRACE nova     super(ServiceLauncher, self).wait()
2012-12-18 12:38:16 TRACE nova   File
"/usr/lib/python2.7/dist-packages/nova/service.py", line 162, in wait
2012-12-18 12:38:16 TRACE nova     service.wait()
2012-12-18 12:38:16 TRACE nova   File
"/usr/lib/python2.7/dist-packages/eventlet/greenthread.py", line 166, in
wait
2012-12-18 12:38:16 TRACE nova     return self._exit_event.wait()
2012-12-18 12:38:16 TRACE nova   File
"/usr/lib/python2.7/dist-packages/eventlet/event.py", line 116, in wait
2012-12-18 12:38:16 TRACE nova     return hubs.get_hub().switch()
2012-12-18 12:38:16 TRACE nova   File
"/usr/lib/python2.7/dist-packages/eventlet/hubs/hub.py", line 177, in switch
2012-12-18 12:38:16 TRACE nova     return self.greenlet.switch()
2012-12-18 12:38:16 TRACE nova   File
"/usr/lib/python2.7/dist-packages/eventlet/greenthread.py", line 192, in
main
2012-12-18 12:38:16 TRACE nova     result = function(*args, **kwargs)
2012-12-18 12:38:16 TRACE nova   File
"/usr/lib/python2.7/dist-packages/nova/service.py", line 132, in run_server
2012-12-18 12:38:16 TRACE nova     server.start()
2012-12-18 12:38:16 TRACE nova   File
"/usr/lib/python2.7/dist-packages/nova/service.py", line 410, in start
2012-12-18 12:38:16 TRACE nova
self.manager.update_available_resource(ctxt)
2012-12-18 12:38:16 TRACE nova   File
"/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 2734, in
update_available_resource
2012-12-18 12:38:16 TRACE nova
self.resource_tracker.update_available_resource(context)
2012-12-18 12:38:16 TRACE nova   File
"/usr/lib/python2.7/dist-packages/nova/utils.py", line 752, in inner
2012-12-18 12:38:16 TRACE nova     retval = f(*args, **kwargs)
2012-12-18 12:38:16 TRACE nova   File
"/usr/lib/python2.7/dist-packages/nova/compute/resource_tracker.py", line
366, in update_available_resource
2012-12-18 12:38:16 TRACE nova     resources =
self.driver.get_available_resource()
2012-12-18 12:38:16 TRACE nova   File
"/usr/lib/python2.7/dist-packages/nova/virt/libvirt/driver.py", line 2212,
in get_available_resource
2012-12-18 12:38:16 TRACE nova     'disk_available_least':
self.get_disk_available_least()}
2012-12-18 12:38:16 TRACE nova   File
"/usr/lib/python2.7/dist-packages/nova/virt/libvirt/driver.py", line 2704,
in get_disk_available_least
2012-12-18 12:38:16 TRACE nova     self.get_instance_disk_info(i_name))
2012-12-18 12:38:16 TRACE nova   File
"/usr/lib/python2.7/dist-packages/nova/virt/libvirt/driver.py", line 2672,
in get_instance_disk_info
2012-12-18 12:38:16 TRACE nova     backing_file =
libvirt_utils.get_disk_backing_file(path)
2012-12-18 12:38:16 TRACE nova   File
"/usr/lib/python2.7/dist-packages/nova/virt/libvirt/utils.py", line 202, in
get_disk_backing_file
2012-12-18 12:38:16 TRACE nova     backing_file =
images.qemu_img_info(path).get('backing file')
2012-12-18 12:38:16 TRACE nova   File
"/usr/lib/python2.7/dist-packages/nova/virt/images.py", line 56, in
qemu_img_info
2012-12-18 12:38:16 TRACE nova     if val[0] == " ":
2012-12-18 12:38:16 TRACE nova IndexError: string index out of range
2012-12-18 12:38:16 TRACE nova