yahoo-eng-team team mailing list archive
-
yahoo-eng-team team
-
Mailing list archive
-
Message #12314
[Bug 1279881] Re: Cannot send scheduler.run_instance events
** Changed in: cinder
Status: Fix Committed => Fix Released
--
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to OpenStack Compute (nova).
https://bugs.launchpad.net/bugs/1279881
Title:
Cannot send scheduler.run_instance events
Status in Cinder:
Fix Released
Status in OpenStack Compute (Nova):
Fix Released
Bug description:
When launching an instance I'm able to trigger the following error
with notifications active:
2014-02-13 16:26:39.974 ERROR oslo.messaging.notify._impl_messaging
[-] Could not send notification to notifications.
Payload={'_context_roles': [u'admin'], '_context_request_id': u'req-
7d0ca56f-f609-40ef-91e1-acf7d5a4a607', '_context_quota_class': None,
'event_type': 'scheduler.run_instance', '_context_service_catalog':
[{u'endpoints': [{u'adminURL':
u'http://162.209.87.220:8776/v1/8c5cfa81726043afb5641ec3d6665f27',
u'region': u'RegionOne', u'id': u'125bb6de586c435eb4e14e5e62090b43',
u'internalURL':
u'http://162.209.87.220:8776/v1/8c5cfa81726043afb5641ec3d6665f27',
u'publicURL':
u'http://162.209.87.220:8776/v1/8c5cfa81726043afb5641ec3d6665f27'}],
u'endpoints_links': [], u'type': u'volume', u'name': u'cinder'}],
'timestamp': '2014-02-13 16:26:39.970161', '_context_user':
u'b6bbfab520dc4f6b91a95cd819f0316c', '_unique_id':
'447230aa40904dd1aa2b6a9319b57f56', '_context_instance_lock_checked':
False, '_context_user_id': u'b6bbfab520dc4f6b91a95cd819f0316c',
'payload': {'instance_id': u'84a4f45c-3f85-4527-ae85-6f323e136d01',
'state': 'error', 'request_spec': {u'num_instances': 1,
u'block_device_mapping': [{u'instance_uuid': u'84a4f45c-
3f85-4527-ae85-6f323e136d01', u'guest_format': None, u'boot_index': 0,
u'delete_on_termination': True, u'no_device': None,
u'connection_info': None, u'snapshot_id': None, u'device_name': None,
u'disk_bus': None, u'image_id': u'0fbb3b3b-
7d69-42d2-8bc7-3b87073e69b2', u'source_type': u'image',
u'device_type': u'disk', u'volume_id': None, u'destination_type':
u'local', u'volume_size': None}], u'image': {u'status': u'active',
u'name': u'ubuntu1310', u'deleted': False, u'container_format':
u'bare', u'created_at': u'2014-02-13T15:50:11.000000', u'disk_format':
u'raw', u'updated_at': u'2014-02-13T15:50:17.000000', u'id': u
'0fbb3b3b-7d69-42d2-8bc7-3b87073e69b2', u'owner':
u'8c5cfa81726043afb5641ec3d6665f27', u'min_ram': 0, u'checksum':
u'5c7a196274f24cd40100eac98f661057', u'min_disk': 0, u'is_public':
True, u'deleted_at': None, u'properties': {}, u'size': 244711424},
u'instance_type': {u'root_gb': 40, u'name': u'm1.medium',
u'ephemeral_gb': 0, u'memory_mb': 4096, u'vcpus': 2, u'extra_specs':
{}, u'swap': 0, u'rxtx_factor': 1.0, u'flavorid': u'3',
u'vcpu_weight': None, u'id': 1}, u'instance_properties': {u'vm_state':
u'building', u'availability_zone': u'nova', u'terminated_at': None,
u'ephemeral_gb': 0, u'instance_type_id': 1, u'user_data': None,
u'cleaned': False, u'vm_mode': None, u'deleted_at': None,
u'reservation_id': u'r-6sv4oy43', u'id': 2, u'security_groups':
[{u'project_id': u'8c5cfa81726043afb5641ec3d6665f27', u'user_id':
u'b6bbfab520dc4f6b91a95cd819f0316c', u'description': u'default',
u'deleted': False, u'created_at': u'2014-02-13T15:50:52.000000',
u'updated_at': None, u'deleted_at': None, u'id': 1, u'name':
u'default'}], u'disable_terminate': False, u'display_name': u'lol',
u'uuid': u'84a4f45c-3f85-4527-ae85-6f323e136d01',
u'default_swap_device': None, u'info_cache': {u'instance_uuid': u
'84a4f45c-3f85-4527-ae85-6f323e136d01', u'deleted': False,
u'created_at': u'2014-02-13T16:26:39.000000', u'updated_at': None,
u'network_info': [], u'deleted_at': None}, u'hostname': u'lol',
u'launched_on': None, u'display_description': u'lol', u'key_data':
None, u'kernel_id': u'', u'power_state': 0,
u'default_ephemeral_device': None, u'progress': 0, u'project_id':
u'8c5cfa81726043afb5641ec3d6665f27', u'launched_at': None,
u'scheduled_at': None, u'node': None, u'ramdisk_id': u'',
u'access_ip_v6': None, u'access_ip_v4': None, u'deleted': False,
u'key_name': None, u'updated_at': None, u'host': None,
u'ephemeral_key_uuid': None, u'architecture': None, u'user_id':
u'b6bbfab520dc4f6b91a95cd819f0316c', u'system_metadata':
{u'image_min_disk': u'40', u'instance_type_memory_mb': u'4096',
u'instance_type_swap': u'0', u'instance_type_vcpu_weight': None,
u'instance_type_root_gb': u'40', u'instance_type_id': u'1',
u'instance_type_name': u'm1.medium', u'instance_type_ephemeral_gb':
u'0', u'instance_type_rxtx_factor': u'1.0', u'instance_type_flavorid':
u'3', u'image_container_format': u'bare', u'instance_type_vcpus':
u'2', u'image_min_ram': u'0', u'image_disk_format': u'raw',
u'image_base_image_ref': u'0fbb3b3b-7d69-42d2-8bc7-3b87073e69b2'},
u'task_state': u'scheduling', u'shutdown_terminate': False,
u'cell_name': None, u'root_gb': 40, u'locked': False, u'name':
u'instance-00000002', u'created_at': u'2014-02-13T16:26:39.000000',
u'locked_by': None, u'launch_index': 0, u'metadata': {}, u'memory_mb':
4096, u'vcpus': 2, u'image_ref': u'0fbb3b3b-
7d69-42d2-8bc7-3b87073e69b2', u'root_device_name': None,
u'auto_disk_config': False, u'os_type': None, u'config_drive': u''},
u'security_group': [u'default']}, 'instance_properties': {u'vm_state':
u'building', u'availability_zone': u'nova', u'terminated_at': None,
u'ephemeral_gb': 0, u'instance_type_id': 1, u'user_data': None,
u'cleaned': False, u'vm_mode': None, u'deleted_at': None,
u'reservation_id': u'r-6sv4oy43', u'id': 2, u'security_groups':
[{u'project_id': u'8c5cfa81726043afb5641ec3d6665f27', u'user_id':
u'b6bbfab520dc4f6b91a95cd819f0316c', u'description': u'default',
u'deleted': False, u'created_at': u'2014-02-13T15:50:52.000000',
u'updated_at': None, u'deleted_at': None, u'id': 1, u'name':
u'default'}], u'disable_terminate': False, u'display_name': u'lol',
u'uuid': u'84a4f45c-3f85-4527-ae85-6f323e136d01',
u'default_swap_device': None, u'info_cache': {u'instance_uuid': u
'84a4f45c-3f85-4527-ae85-6f323e136d01', u'deleted': False,
u'created_at': u'2014-02-13T16:26:39.000000', u'updated_at': None,
u'network_info': [], u'deleted_at': None}, u'hostname': u'lol',
u'launched_on': None, u'display_description': u'lol', u'key_data':
None, u'kernel_id': u'', u'power_state': 0,
u'default_ephemeral_device': None, u'progress': 0, u'project_id':
u'8c5cfa81726043afb5641ec3d6665f27', u'launched_at': None,
u'scheduled_at': None, u'node': None, u'ramdisk_id': u'',
u'access_ip_v6': None, u'access_ip_v4': None, u'deleted': False,
u'key_name': None, u'updated_at': None, u'host': None,
u'ephemeral_key_uuid': None, u'architecture': None, u'user_id':
u'b6bbfab520dc4f6b91a95cd819f0316c', u'system_metadata':
{u'image_min_disk': u'40', u'instance_type_memory_mb': u'4096',
u'instance_type_swap': u'0', u'instance_type_vcpu_weight': None,
u'instance_type_root_gb': u'40', u'instance_type_id': u'1',
u'instance_type_name': u'm1.medium', u'instance_type_ephemeral_gb':
u'0', u'instance_type_rxtx_factor': u'1.0', u'instance_type_flavorid':
u'3', u'image_container_format': u'bare', u'instance_type_vcpus':
u'2', u'image_min_ram': u'0', u'image_disk_format': u'raw',
u'image_base_image_ref': u'0fbb3b3b-7d69-42d2-8bc7-3b87073e69b2'},
u'task_state': u'scheduling', u'shutdown_terminate': False,
u'cell_name': None, u'root_gb': 40, u'locked': False, u'name':
u'instance-00000002', u'created_at': u'2014-02-13T16:26:39.000000',
u'locked_by': None, u'launch_index': 0, u'metadata': {}, u'memory_mb':
4096, u'vcpus': 2, u'image_ref': u'0fbb3b3b-
7d69-42d2-8bc7-3b87073e69b2', u'root_device_name': None,
u'auto_disk_config': False, u'os_type': None, u'config_drive': u''},
'reason': NoValidHost(u'No valid host was found. ',), 'method':
'run_instance'}, '_context_project_name': u'demo',
'_context_read_deleted': u'no', '_context_auth_token':
u'06fb3c10469c7811eeae6936ff3b12ec', '_context_tenant':
u'8c5cfa81726043afb5641ec3d6665f27', 'priority': 'ERROR',
'_context_is_admin': True, '_context_project_id':
u'8c5cfa81726043afb5641ec3d6665f27', '_context_timestamp':
'2014-02-13T16:26:39.446679', '_context_user_name': u'admin',
'publisher_id': 'scheduler.devstack', 'message_id':
'f1f2ab74-1bd7-4992-89e7-93de9adeae5a', '_context_remote_address':
u'162.209.87.220'}
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging Traceback (most recent call last):
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging File "/opt/stack/oslo.messaging/oslo/messaging/notify/_impl_messaging.py", line 47, in notify
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging version=self.version)
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging File "/opt/stack/oslo.messaging/oslo/messaging/transport.py", line 93, in _send_notification
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging self._driver.send_notification(target, ctxt, message, version)
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging File "/opt/stack/oslo.messaging/oslo/messaging/_drivers/amqpdriver.py", line 393, in send_notification
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging return self._send(target, ctxt, message, envelope=(version == 2.0))
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging File "/opt/stack/oslo.messaging/oslo/messaging/_drivers/amqpdriver.py", line 378, in _send
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging conn.topic_send(topic, msg, timeout=timeout)
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging File "/opt/stack/oslo.messaging/oslo/messaging/_drivers/impl_rabbit.py", line 701, in topic_send
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging self.publisher_send(TopicPublisher, topic, msg, timeout)
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging File "/opt/stack/oslo.messaging/oslo/messaging/_drivers/impl_rabbit.py", line 673, in publisher_send
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging self.ensure(_error_callback, _publish)
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging File "/opt/stack/oslo.messaging/oslo/messaging/_drivers/impl_rabbit.py", line 577, in ensure
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging return method(*args, **kwargs)
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging File "/opt/stack/oslo.messaging/oslo/messaging/_drivers/impl_rabbit.py", line 671, in _publish
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging publisher.send(msg, timeout)
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging File "/opt/stack/oslo.messaging/oslo/messaging/_drivers/impl_rabbit.py", line 328, in send
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging self.producer.publish(msg)
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging File "/usr/local/lib/python2.7/dist-packages/kombu/messaging.py", line 157, in publish
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging compression, headers)
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging File "/usr/local/lib/python2.7/dist-packages/kombu/messaging.py", line 233, in _prepare
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging body) = encode(body, serializer=serializer)
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging File "/usr/local/lib/python2.7/dist-packages/kombu/serialization.py", line 170, in encode
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging payload = encoder(data)
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging File "/usr/lib/python2.7/dist-packages/anyjson/__init__.py", line 141, in dumps
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging return implementation.dumps(value)
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging File "/usr/lib/python2.7/dist-packages/anyjson/__init__.py", line 87, in dumps
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging return self._encode(data)
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging File "/usr/local/lib/python2.7/dist-packages/neutronclient/openstack/common/jsonutils.py", line 155, in dumps
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging return json.dumps(value, default=default, **kwargs)
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging File "/usr/lib/python2.7/json/__init__.py", line 250, in dumps
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging sort_keys=sort_keys, **kw).encode(obj)
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging File "/usr/lib/python2.7/json/encoder.py", line 207, in encode
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging chunks = self.iterencode(o, _one_shot=True)
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging File "/usr/lib/python2.7/json/encoder.py", line 270, in iterencode
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging return _iterencode(o, 0)
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging ValueError: Circular reference detected
2014-02-13 16:26:39.974 TRACE oslo.messaging.notify._impl_messaging
To manage notifications about this bug go to:
https://bugs.launchpad.net/cinder/+bug/1279881/+subscriptions
References
