yahoo-eng-team team mailing list archive
-
yahoo-eng-team team
-
Mailing list archive
-
Message #23748
[Bug 1381295] [NEW] live-migration with context by get_admin_context would fail
Public bug reported:
when 3rd-party wants to live-migration a VM to a host by call compute
API directly ( not by novaclient ) like code snippet 1 as below, the
destination compute service would call glance to get image(refer to Code
snippet 2 as below), the destniation compute service would raise
exception as Log snippet 1 as below, this is caused by patch
https://review.openstack.org/#/c/121692/2. I think similar problem
exists in nova, too, refer to Code snippet 3.
Code snippet 1:
from nova import compute
ctxt = context.get_admin_context()
self.compute_api = compute.API()
self.compute_api.live_migrate(
ctxt.elevated(), inst, False, False, host_dict)
Code snippet 2:nova.image.glance.py:
def _create_glance_client(context, host, port, use_ssl, version=1):
"""Instantiate a new glanceclient.Client object."""
params = {}
if use_ssl:
scheme = 'https'
# https specific params
params['insecure'] = CONF.glance.api_insecure
params['ssl_compression'] = False
if CONF.ssl.cert_file:
params['cert_file'] = CONF.ssl.cert_file
if CONF.ssl.key_file:
params['key_file'] = CONF.ssl.key_file
if CONF.ssl.ca_file:
params['cacert'] = CONF.ssl.ca_file
else:
scheme = 'http'
if CONF.auth_strategy == 'keystone':
# NOTE(isethi): Glanceclient <= 0.9.0.49 accepts only
# keyword 'token', but later versions accept both the
# header 'X-Auth-Token' and 'token'
params['token'] = context.auth_token
params['identity_headers'] = generate_identity_headers(context) <<<<<<<<<would return {'X-Auth-Token': None, ....}
if utils.is_valid_ipv6(host):
# if so, it is ipv6 address, need to wrap it with '[]'
host = '[%s]' % host
endpoint = '%s://%s:%s' % (scheme, host, port)
return glanceclient.Client(str(version), endpoint, **params) <<<<<<<<<<<<<<<params=={'identity_headers':{{'X-Auth-Token': None, ....}}...}
Code snippet 3:
novaclient.client.py:
def http_log_req(self, method, url, kwargs):
if not self.http_log_debug:
return
string_parts = ['curl -i']
if not kwargs.get('verify', True):
string_parts.append(' --insecure')
string_parts.append(" '%s'" % url)
string_parts.append(' -X %s' % method)
headers = copy.deepcopy(kwargs['headers'])
self._redact(headers, ['X-Auth-Token']) >>>>>>>>>>>>>>>>>>>here
# because dict ordering changes from 2 to 3
keys = sorted(headers.keys())
for name in keys:
value = headers[name]
header = ' -H "%s: %s"' % (name, value)
string_parts.append(header)
if 'data' in kwargs:
data = json.loads(kwargs['data'])
self._redact(data, ['auth', 'passwordCredentials', 'password'])
string_parts.append(" -d '%s'" % json.dumps(data))
self._logger.debug("REQ: %s" % "".join(string_parts))
Log snippet 1:
2014-10-14 00:42:10.699 31346 INFO nova.compute.manager [-] [instance: aa68237f-e669-4025-b16e-f4b50926f7a5] During the sync_power process the instance has moved from host cmo-comp5.ibm.com to host cmo-comp4.ibm.com
2014-10-14 00:42:10.913 31346 INFO nova.compute.manager [req-7be58838-3ec2-43d4-afd1-23d6b3d5e3de None] [instance: aa68237f-e669-4025-b16e-f4b50926f7a5] Post operation of migration started
2014-10-14 00:42:11.148 31346 ERROR oslo.messaging.rpc.dispatcher [req-7be58838-3ec2-43d4-afd1-23d6b3d5e3de ] Exception during message handling: 'NoneType' object has no attribute 'encode'
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher Traceback (most recent call last):
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/oslo/messaging/rpc/dispatcher.py", line 134, in _dispatch_and_reply
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher incoming.message))
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/oslo/messaging/rpc/dispatcher.py", line 177, in _dispatch
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher return self._do_dispatch(endpoint, method, ctxt, args)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/oslo/messaging/rpc/dispatcher.py", line 123, in _do_dispatch
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher result = getattr(endpoint, method)(ctxt, **new_args)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 418, in decorated_function
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher return function(self, context, *args, **kwargs)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/exception.py", line 88, in wrapped
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher payload)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py", line 82, in __exit__
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher six.reraise(self.type_, self.value, self.tb)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/exception.py", line 71, in wrapped
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher return f(self, context, *args, **kw)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 330, in decorated_function
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher kwargs['instance'], e, sys.exc_info())
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py", line 82, in __exit__
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher six.reraise(self.type_, self.value, self.tb)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 318, in decorated_function
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher return function(self, context, *args, **kwargs)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 5224, in post_live_migration_at_destination
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher block_migration, block_device_info)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/virt/libvirt/driver.py", line 5635, in post_live_migration_at_destination
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher write_to_disk=True)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/virt/libvirt/driver.py", line 4132, in _get_guest_xml
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher context, self._image_api, image_ref, instance)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/utils.py", line 242, in get_image_metadata
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher image = image_api.get(context, image_id_or_uri)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/image/api.py", line 89, in get
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher include_locations=include_locations)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/image/glance.py", line 311, in show
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher _reraise_translated_image_exception(image_id)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/image/glance.py", line 309, in show
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher image = self._client.call(context, version, 'get', image_id)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/image/glance.py", line 232, in call
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher return getattr(client.images, method)(*args, **kwargs)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/glanceclient/v1/images.py", line 126, in get
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher % urlparse.quote(str(image_id)))
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/glanceclient/common/http.py", line 248, in head
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher return self._request('HEAD', url, **kwargs)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/glanceclient/common/http.py", line 192, in _request
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher self.log_curl_request(method, conn_url, headers, data, kwargs)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/glanceclient/common/http.py", line 99, in log_curl_request
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher header = '-H \'%s: %s\'' % safe_header(key, value)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/glanceclient/common/utils.py", line 394, in safe_header
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher v = value.encode('utf-8')
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher AttributeError: 'NoneType' object has no attribute 'encode'
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher
2014-10-14 00:42:11.152 31346 ERROR oslo.messaging._drivers.common [req-7be58838-3ec2-43d4-afd1-23d6b3d5e3de ] Returning exception 'NoneType' object has no attribute 'encode' to caller
** Affects: nova
Importance: Undecided
Status: New
--
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to OpenStack Compute (nova).
https://bugs.launchpad.net/bugs/1381295
Title:
live-migration with context by get_admin_context would fail
Status in OpenStack Compute (Nova):
New
Bug description:
when 3rd-party wants to live-migration a VM to a host by call compute
API directly ( not by novaclient ) like code snippet 1 as below, the
destination compute service would call glance to get image(refer to
Code snippet 2 as below), the destniation compute service would raise
exception as Log snippet 1 as below, this is caused by patch
https://review.openstack.org/#/c/121692/2. I think similar problem
exists in nova, too, refer to Code snippet 3.
Code snippet 1:
from nova import compute
ctxt = context.get_admin_context()
self.compute_api = compute.API()
self.compute_api.live_migrate(
ctxt.elevated(), inst, False, False, host_dict)
Code snippet 2:nova.image.glance.py:
def _create_glance_client(context, host, port, use_ssl, version=1):
"""Instantiate a new glanceclient.Client object."""
params = {}
if use_ssl:
scheme = 'https'
# https specific params
params['insecure'] = CONF.glance.api_insecure
params['ssl_compression'] = False
if CONF.ssl.cert_file:
params['cert_file'] = CONF.ssl.cert_file
if CONF.ssl.key_file:
params['key_file'] = CONF.ssl.key_file
if CONF.ssl.ca_file:
params['cacert'] = CONF.ssl.ca_file
else:
scheme = 'http'
if CONF.auth_strategy == 'keystone':
# NOTE(isethi): Glanceclient <= 0.9.0.49 accepts only
# keyword 'token', but later versions accept both the
# header 'X-Auth-Token' and 'token'
params['token'] = context.auth_token
params['identity_headers'] = generate_identity_headers(context) <<<<<<<<<would return {'X-Auth-Token': None, ....}
if utils.is_valid_ipv6(host):
# if so, it is ipv6 address, need to wrap it with '[]'
host = '[%s]' % host
endpoint = '%s://%s:%s' % (scheme, host, port)
return glanceclient.Client(str(version), endpoint, **params) <<<<<<<<<<<<<<<params=={'identity_headers':{{'X-Auth-Token': None, ....}}...}
Code snippet 3:
novaclient.client.py:
def http_log_req(self, method, url, kwargs):
if not self.http_log_debug:
return
string_parts = ['curl -i']
if not kwargs.get('verify', True):
string_parts.append(' --insecure')
string_parts.append(" '%s'" % url)
string_parts.append(' -X %s' % method)
headers = copy.deepcopy(kwargs['headers'])
self._redact(headers, ['X-Auth-Token']) >>>>>>>>>>>>>>>>>>>here
# because dict ordering changes from 2 to 3
keys = sorted(headers.keys())
for name in keys:
value = headers[name]
header = ' -H "%s: %s"' % (name, value)
string_parts.append(header)
if 'data' in kwargs:
data = json.loads(kwargs['data'])
self._redact(data, ['auth', 'passwordCredentials', 'password'])
string_parts.append(" -d '%s'" % json.dumps(data))
self._logger.debug("REQ: %s" % "".join(string_parts))
Log snippet 1:
2014-10-14 00:42:10.699 31346 INFO nova.compute.manager [-] [instance: aa68237f-e669-4025-b16e-f4b50926f7a5] During the sync_power process the instance has moved from host cmo-comp5.ibm.com to host cmo-comp4.ibm.com
2014-10-14 00:42:10.913 31346 INFO nova.compute.manager [req-7be58838-3ec2-43d4-afd1-23d6b3d5e3de None] [instance: aa68237f-e669-4025-b16e-f4b50926f7a5] Post operation of migration started
2014-10-14 00:42:11.148 31346 ERROR oslo.messaging.rpc.dispatcher [req-7be58838-3ec2-43d4-afd1-23d6b3d5e3de ] Exception during message handling: 'NoneType' object has no attribute 'encode'
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher Traceback (most recent call last):
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/oslo/messaging/rpc/dispatcher.py", line 134, in _dispatch_and_reply
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher incoming.message))
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/oslo/messaging/rpc/dispatcher.py", line 177, in _dispatch
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher return self._do_dispatch(endpoint, method, ctxt, args)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/oslo/messaging/rpc/dispatcher.py", line 123, in _do_dispatch
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher result = getattr(endpoint, method)(ctxt, **new_args)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 418, in decorated_function
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher return function(self, context, *args, **kwargs)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/exception.py", line 88, in wrapped
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher payload)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py", line 82, in __exit__
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher six.reraise(self.type_, self.value, self.tb)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/exception.py", line 71, in wrapped
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher return f(self, context, *args, **kw)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 330, in decorated_function
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher kwargs['instance'], e, sys.exc_info())
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/openstack/common/excutils.py", line 82, in __exit__
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher six.reraise(self.type_, self.value, self.tb)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 318, in decorated_function
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher return function(self, context, *args, **kwargs)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/manager.py", line 5224, in post_live_migration_at_destination
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher block_migration, block_device_info)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/virt/libvirt/driver.py", line 5635, in post_live_migration_at_destination
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher write_to_disk=True)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/virt/libvirt/driver.py", line 4132, in _get_guest_xml
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher context, self._image_api, image_ref, instance)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/compute/utils.py", line 242, in get_image_metadata
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher image = image_api.get(context, image_id_or_uri)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/image/api.py", line 89, in get
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher include_locations=include_locations)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/image/glance.py", line 311, in show
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher _reraise_translated_image_exception(image_id)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/image/glance.py", line 309, in show
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher image = self._client.call(context, version, 'get', image_id)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/nova/image/glance.py", line 232, in call
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher return getattr(client.images, method)(*args, **kwargs)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/glanceclient/v1/images.py", line 126, in get
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher % urlparse.quote(str(image_id)))
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/glanceclient/common/http.py", line 248, in head
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher return self._request('HEAD', url, **kwargs)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/glanceclient/common/http.py", line 192, in _request
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher self.log_curl_request(method, conn_url, headers, data, kwargs)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/glanceclient/common/http.py", line 99, in log_curl_request
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher header = '-H \'%s: %s\'' % safe_header(key, value)
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.6/site-packages/glanceclient/common/utils.py", line 394, in safe_header
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher v = value.encode('utf-8')
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher AttributeError: 'NoneType' object has no attribute 'encode'
2014-10-14 00:42:11.148 31346 TRACE oslo.messaging.rpc.dispatcher
2014-10-14 00:42:11.152 31346 ERROR oslo.messaging._drivers.common [req-7be58838-3ec2-43d4-afd1-23d6b3d5e3de ] Returning exception 'NoneType' object has no attribute 'encode' to caller
To manage notifications about this bug go to:
https://bugs.launchpad.net/nova/+bug/1381295/+subscriptions
Follow ups
References