← Back to team overview

yahoo-eng-team team mailing list archive

  1. yahoo-eng-team team
  2. Mailing list archive
  3. Message #16586

[Bug 1334024] [NEW] Nova rescue fails for libvirt driver with config drive

  Thread Previous • Date Previous • • Thread Next 
Public bug reported:

I am using config drive to boot VMs. In icehouse, I observed that nova
rescue fails and leaves the VM in SHUTOFF state.

Short error log: 
/home/y/var/nova/instances/270e299b-90b2-46d5-bf9a-e7f6efe3742e/disk.config.rescue': No such file or directory

Difference in Havana and Icehouse code path:

# Havana
# Config drive
        if configdrive.required_by(instance):
            LOG.info(_('Using config drive'), instance=instance)
            extra_md = {}
            if admin_pass:
                extra_md['admin_pass'] = admin_pass

            for f in ('user_name', 'project_name'):
                if hasattr(context, f):
                    extra_md[f] = getattr(context, f, None)
            inst_md = instance_metadata.InstanceMetadata(instance,
                content=files, extra_md=extra_md, network_info=network_info)
            with configdrive.ConfigDriveBuilder(instance_md=inst_md) as cdb:
                configdrive_path = basepath(fname='disk.config')
                               LOG.info(_('Creating config drive at %(path)s'),
                         {'path': configdrive_path}, instance=instance)


        def basepath(fname='', suffix=suffix): << Adds suffix .rescue to disk.config.
            return os.path.join(libvirt_utils.get_instance_path(instance),
                                fname + suffix)


# Icehouse:

# Config drive
        if configdrive.required_by(instance):
            LOG.info(_('Using config drive'), instance=instance)
            extra_md = {}
            if admin_pass:
                extra_md['admin_pass'] = admin_pass

            for f in ('user_name', 'project_name'):
                if hasattr(context, f):
                    extra_md[f] = getattr(context, f, None)
            inst_md = instance_metadata.InstanceMetadata(instance,
                content=files, extra_md=extra_md, network_info=network_info)
            with configdrive.ConfigDriveBuilder(instance_md=inst_md) as cdb:
                configdrive_path = self._get_disk_config_path(instance)
                              LOG.info(_('Creating config drive at %(path)s'),
                         {'path': configdrive_path}, instance=instance)

     @staticmethod
    def _get_disk_config_path(instance):
        return os.path.join(libvirt_utils.get_instance_path(instance),
                            'disk.config')

The suffix .rescue is missed here and hence, original disk.config is
overwritten.

Following change fixed the issue for me:

configdrive_path = self._get_disk_config_path(instance, suffix)
 
@staticmethod
    def _get_disk_config_path(instance, suffix=''):
        return os.path.join(libvirt_utils.get_instance_path(instance),
                            'disk.config' + suffix)


Complete Error log:

{"extra": {"project_name": "admin", "timestamp": "2014-06-20T23:00:50.269421",
"auth_token": "17fcde000c3040f9981e1804cdaf94fe", "remote_address":
"10.220.4.45", "quota_class": null, "is_admin": true, "user":
"dfac8c9e704a4312b0447b26b57a12da", "service_catalog": [], "tenant":
"13b05759646b41c9a51660a1e653b146", "user_id":
"dfac8c9e704a4312b0447b26b57a12da", "roles": ["admin"], "project":
"nova.compute.manager", "instance": "[instance:
270e299b-90b2-46d5-bf9a-e7f6efe3742e] ", "version": "unknown", "read_deleted":
"no", "request_id": "req-a7f12cd9-dd23-48fa-9479-ab4e7825ae1e",
"instance_lock_checked": false, "project_id":
"13b05759646b41c9a51660a1e653b146", "user_name": "spandhe"}, "thread_name":
"GreenThread-31", "process_name": "MainProcess", "name":
"nova.compute.manager", "thread": 82908816, "created": 1403305252.7287409,
"process": 26178, "relative_created": 114349619.65203285, "args": [],
"traceback": ["Traceback (most recent call last):", "  File
\"/usr/lib/python2.6/site-packages/nova/compute/manager.py\", line 2983, in
rescue_instance", "    rescue_image_meta, admin_password)", "  File
\"/usr/lib/python2.6/site-packages/nova/virt/libvirt/driver.py\", line 2205, in
rescue", "    self._create_domain(xml)", "  File
\"/usr/lib/python2.6/site-packages/nova/virt/libvirt/driver.py\", line 3562, in
_create_domain", "    domain.XMLDesc(0))", "  File
\"/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py\", line
68, in __exit__", "    six.reraise(self.type_, self.value, self.tb)", "  File
\"/usr/lib/python2.6/site-packages/nova/virt/libvirt/driver.py\", line 3557, in
_create_domain", "    domain.createWithFlags(launch_flags)", "  File
\"/usr/lib/python2.6/site-packages/eventlet/tpool.py\", line 179, in doit", "  
 result = proxy_call(self._autowrap, f, *args, **kwargs)", "  File
\"/usr/lib/python2.6/site-packages/eventlet/tpool.py\", line 139, in
proxy_call", "    rv = execute(f,*args,**kwargs)", "  File
\"/usr/lib/python2.6/site-packages/eventlet/tpool.py\", line 77, in tworker", "
   rv = meth(*args,**kwargs)", "  File
\"/usr/lib64/python2.6/site-packages/libvirt.py\", line 708, in
createWithFlags", "    if ret == -1: raise libvirtError
('virDomainCreateWithFlags() failed', dom=self)", "libvirtError: cannot open
file
'/home/y/var/nova/instances/270e299b-90b2-46d5-bf9a-e7f6efe3742e/disk.config.rescue':
No such file or directory"], "module": "manager", "filename": "manager.py",
"levelno": 40, "msecs": 728.74093055725098, "pathname":
"/usr/lib/python2.6/site-packages/nova/compute/manager.py", "lineno": 2986,
"asctime": "2014-06-20 23:00:52,728", "msg": "Error trying to Rescue Instance",
"message": "Error trying to Rescue Instance", "funcname": "rescue_instance",
"levelname": "ERROR"}

** Affects: nova
     Importance: Undecided
     Assignee: Shraddha Pandhe (shraddha-pandhe)
         Status: New

** Changed in: nova
     Assignee: (unassigned) => Shraddha Pandhe (shraddha-pandhe)

-- 
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to OpenStack Compute (nova).
https://bugs.launchpad.net/bugs/1334024

Title:
  Nova rescue fails for libvirt driver with config drive

Status in OpenStack Compute (Nova):
  New

Bug description:
  I am using config drive to boot VMs. In icehouse, I observed that nova
  rescue fails and leaves the VM in SHUTOFF state.

  Short error log: 
  /home/y/var/nova/instances/270e299b-90b2-46d5-bf9a-e7f6efe3742e/disk.config.rescue': No such file or directory

  Difference in Havana and Icehouse code path:

  # Havana
  # Config drive
          if configdrive.required_by(instance):
              LOG.info(_('Using config drive'), instance=instance)
              extra_md = {}
              if admin_pass:
                  extra_md['admin_pass'] = admin_pass

              for f in ('user_name', 'project_name'):
                  if hasattr(context, f):
                      extra_md[f] = getattr(context, f, None)
              inst_md = instance_metadata.InstanceMetadata(instance,
                  content=files, extra_md=extra_md, network_info=network_info)
              with configdrive.ConfigDriveBuilder(instance_md=inst_md) as cdb:
                  configdrive_path = basepath(fname='disk.config')
                                 LOG.info(_('Creating config drive at %(path)s'),
                           {'path': configdrive_path}, instance=instance)

  
          def basepath(fname='', suffix=suffix): << Adds suffix .rescue to disk.config.
              return os.path.join(libvirt_utils.get_instance_path(instance),
                                  fname + suffix)

  
  # Icehouse:

  # Config drive
          if configdrive.required_by(instance):
              LOG.info(_('Using config drive'), instance=instance)
              extra_md = {}
              if admin_pass:
                  extra_md['admin_pass'] = admin_pass

              for f in ('user_name', 'project_name'):
                  if hasattr(context, f):
                      extra_md[f] = getattr(context, f, None)
              inst_md = instance_metadata.InstanceMetadata(instance,
                  content=files, extra_md=extra_md, network_info=network_info)
              with configdrive.ConfigDriveBuilder(instance_md=inst_md) as cdb:
                  configdrive_path = self._get_disk_config_path(instance)
                                LOG.info(_('Creating config drive at %(path)s'),
                           {'path': configdrive_path}, instance=instance)

       @staticmethod
      def _get_disk_config_path(instance):
          return os.path.join(libvirt_utils.get_instance_path(instance),
                              'disk.config')

  The suffix .rescue is missed here and hence, original disk.config is
  overwritten.

  Following change fixed the issue for me:

  configdrive_path = self._get_disk_config_path(instance, suffix)
   
  @staticmethod
      def _get_disk_config_path(instance, suffix=''):
          return os.path.join(libvirt_utils.get_instance_path(instance),
                              'disk.config' + suffix)

  
  Complete Error log:

  {"extra": {"project_name": "admin", "timestamp": "2014-06-20T23:00:50.269421",
  "auth_token": "17fcde000c3040f9981e1804cdaf94fe", "remote_address":
  "10.220.4.45", "quota_class": null, "is_admin": true, "user":
  "dfac8c9e704a4312b0447b26b57a12da", "service_catalog": [], "tenant":
  "13b05759646b41c9a51660a1e653b146", "user_id":
  "dfac8c9e704a4312b0447b26b57a12da", "roles": ["admin"], "project":
  "nova.compute.manager", "instance": "[instance:
  270e299b-90b2-46d5-bf9a-e7f6efe3742e] ", "version": "unknown", "read_deleted":
  "no", "request_id": "req-a7f12cd9-dd23-48fa-9479-ab4e7825ae1e",
  "instance_lock_checked": false, "project_id":
  "13b05759646b41c9a51660a1e653b146", "user_name": "spandhe"}, "thread_name":
  "GreenThread-31", "process_name": "MainProcess", "name":
  "nova.compute.manager", "thread": 82908816, "created": 1403305252.7287409,
  "process": 26178, "relative_created": 114349619.65203285, "args": [],
  "traceback": ["Traceback (most recent call last):", "  File
  \"/usr/lib/python2.6/site-packages/nova/compute/manager.py\", line 2983, in
  rescue_instance", "    rescue_image_meta, admin_password)", "  File
  \"/usr/lib/python2.6/site-packages/nova/virt/libvirt/driver.py\", line 2205, in
  rescue", "    self._create_domain(xml)", "  File
  \"/usr/lib/python2.6/site-packages/nova/virt/libvirt/driver.py\", line 3562, in
  _create_domain", "    domain.XMLDesc(0))", "  File
  \"/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py\", line
  68, in __exit__", "    six.reraise(self.type_, self.value, self.tb)", "  File
  \"/usr/lib/python2.6/site-packages/nova/virt/libvirt/driver.py\", line 3557, in
  _create_domain", "    domain.createWithFlags(launch_flags)", "  File
  \"/usr/lib/python2.6/site-packages/eventlet/tpool.py\", line 179, in doit", "  
   result = proxy_call(self._autowrap, f, *args, **kwargs)", "  File
  \"/usr/lib/python2.6/site-packages/eventlet/tpool.py\", line 139, in
  proxy_call", "    rv = execute(f,*args,**kwargs)", "  File
  \"/usr/lib/python2.6/site-packages/eventlet/tpool.py\", line 77, in tworker", "
     rv = meth(*args,**kwargs)", "  File
  \"/usr/lib64/python2.6/site-packages/libvirt.py\", line 708, in
  createWithFlags", "    if ret == -1: raise libvirtError
  ('virDomainCreateWithFlags() failed', dom=self)", "libvirtError: cannot open
  file
  '/home/y/var/nova/instances/270e299b-90b2-46d5-bf9a-e7f6efe3742e/disk.config.rescue':
  No such file or directory"], "module": "manager", "filename": "manager.py",
  "levelno": 40, "msecs": 728.74093055725098, "pathname":
  "/usr/lib/python2.6/site-packages/nova/compute/manager.py", "lineno": 2986,
  "asctime": "2014-06-20 23:00:52,728", "msg": "Error trying to Rescue Instance",
  "message": "Error trying to Rescue Instance", "funcname": "rescue_instance",
  "levelname": "ERROR"}

To manage notifications about this bug go to:
https://bugs.launchpad.net/nova/+bug/1334024/+subscriptions

Follow ups

References

  Thread Previous • Date Previous • • Thread Next