yahoo-eng-team team mailing list archive

Thread
Date
[Bug 1800441] [NEW] Unexpected exception in API method: MessagingTimeout: Timed out waiting for a reply to message ID

To: yahoo-eng-team@xxxxxxxxxxxxxxxxxxx
From: chinassh1209 <1800441@xxxxxxxxxxxxxxxxxx>
Date: Mon, 29 Oct 2018 09:47:01 -0000
Reply-to: Bug 1800441 <1800441@xxxxxxxxxxxxxxxxxx>
Sender: bounces@xxxxxxxxxxxxx
Public bug reported:

Hello, I built all of rocky on one mainframe. The problem now is that I
have no access to novnc. Everything else is okay.

Some system versions and information:

[root@all-in-one-202 ~]# cat /etc/centos-release
CentOS Linux release 7.5.1804 (Core)

[root@all-in-one-202 ~]# uname -a
Linux all-in-one-202 3.10.0-862.11.6.el7.x86_64 #1 SMP Tue Aug 14 21:49:04 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux

[root@all-in-one-202 ~]# rpm -qa | grep openstack-rocky
centos-release-openstack-rocky-1-1.el7.centos.noarch

[root@all-in-one-202 ~]# rpm -qa | grep memcache
memcached-1.5.6-1.el7.x86_64
python-memcached-1.58-1.el7.noarch

[root@all-in-one-202 ~]# cat /etc/sysconfig/memcached
PORT="11211"
USER="memcached"
MAXCONN="1024"
CACHESIZE="64"
OPTIONS=""

[root@all-in-one-202 ~]# egrep -v "^#|^$" /etc/nova/nova.conf
[DEFAULT]
enabled_apis = osapi_compute,metadata
transport_url = rabbit://openstack:Admin@10.100.26.202
my_ip = 10.100.26.202
use_neutron = true
firewall_driver = nova.virt.firewall.NoopFirewallDriver
enabled_apis = osapi_compute,metadata
enable_network_quota=false
[api]
auth_strategy = keystone
[api_database]
connection = mysql+pymysql://nova:Admin@10.100.26.202/nova_api
[barbican]
[cache]
memcached_servers=10.100.26.202:11211
[cells]
[cinder]
[compute]
[conductor]
[console]
[consoleauth]
[cors]
[database]
connection = mysql+pymysql://nova:Admin@10.100.26.202/nova
[devices]
[ephemeral_storage_encryption]
[filter_scheduler]
[glance]
api_servers = http://10.100.26.202:9292
[guestfs]
[healthcheck]
[hyperv]
[ironic]
[key_manager]
[keystone]
[keystone_authtoken]
auth_url = http://10.100.26.202:5000/v3
memcached_servers = 10.100.26.202:11211
auth_type = password
project_domain_name = default
user_domain_name = default
project_name = service
username = nova
password = Admin
[libvirt]
virt_type = qemu
[matchmaker_redis]
[metrics]
[mks]
[neutron]
url = http://10.100.26.202:9696
auth_url = http://10.100.26.202:5000
auth_type = password
project_domain_name = default
user_domain_name = default
region_name = RegionOne
project_name = service
username = neutron
password = Admin
service_metadata_proxy = true
metadata_proxy_shared_secret = Admin
[notifications]
[osapi_v21]
[oslo_concurrency]
lock_path = /var/lib/nova/tmp
[oslo_messaging_amqp]
[oslo_messaging_kafka]
[oslo_messaging_notifications]
[oslo_messaging_rabbit]
[oslo_messaging_zmq]
[oslo_middleware]
[oslo_policy]
[pci]
[placement]
region_name = RegionOne
project_domain_name = Default
project_name = service
auth_type = password
user_domain_name = Default
auth_url = http://10.100.26.202:5000/v3
username = placement
password = Admin
[placement_database]
connection = mysql+pymysql://placement:Admin@10.100.26.202/placement
[powervm]
[profiler]
[quota]
instances=-1
cores=-1
ram=-1
floating_ips=-1
metadata_items=-1
injected_files=-1
injected_file_content_bytes=-1
security_groups=-1
security_group_rules=-1
[rdp]
[remote_debug]
[scheduler]
[serial_console]
[service_user]
[spice]
[upgrade_levels]
[vault]
[vendordata_dynamic_auth]
[vmware]
[vnc]
enabled = True
server_listen = 10.100.26.202
server_proxyclient_address = 10.100.26.202
novncproxy_base_url = http://10.100.26.202:6080/vnc_auto.html
[workarounds]
[wsgi]
[xenserver]
[xvp]
[zvm]

[root@all-in-one-202 ~]# tail -200f /var/log/nova/nova-api.log

2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi [req-155d3560-8788-44be-ac3d-84b99baa1196 3e674e62b3cf490abbba13bab06449e2 63ff15cc51b7432a845633576b5183d6 - default default] Unexpected exception in API method: MessagingTimeout: Timed out waiting for a reply to message ID 991b3e95605c4ac6a33ba0a8ed722a4c
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi Traceback (most recent call last):
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/nova/api/openstack/wsgi.py", line 801, in wrapped
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     return f(*args, **kwargs)
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/nova/api/validation/__init__.py", line 110, in wrapper
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     return func(*args, **kwargs)
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/nova/api/openstack/compute/remote_consoles.py", line 52, in get_vnc_console
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     console_type)
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/nova/compute/api.py", line 196, in wrapped
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     return function(self, context, instance, *args, **kwargs)
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/nova/compute/api.py", line 186, in inner
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     return f(self, context, instance, *args, **kw)
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/nova/compute/api.py", line 3750, in get_vnc_console
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     access_url=connect_info['access_url'])
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/nova/consoleauth/rpcapi.py", line 93, in authorize_console
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     return cctxt.call(ctxt, 'authorize_console', **msg_args)
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/oslo_messaging/rpc/client.py", line 179, in call
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     retry=self.retry)
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/oslo_messaging/transport.py", line 133, in _send
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     retry=retry)
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 584, in send
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     call_monitor_timeout, retry=retry)
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 573, in _send
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     call_monitor_timeout)
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 459, in wait
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     message = self.waiters.get(msg_id, timeout=timeout)
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 336, in get
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     'to message ID %s' % msg_id)
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi MessagingTimeout: Timed out waiting for a reply to message ID 991b3e95605c4ac6a33ba0a8ed722a4c
2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi
2018-10-29 17:26:13.449 16284 INFO nova.api.openstack.wsgi [req-155d3560-8788-44be-ac3d-84b99baa1196 3e674e62b3cf490abbba13bab06449e2 63ff15cc51b7432a845633576b5183d6 - default default] HTTP exception thrown: Unexpected API Error. Please report this at http://bugs.launchpad.net/nova/ and attach the Nova API log if possible.
<class 'oslo_messaging.exceptions.MessagingTimeout'>

** Affects: nova
     Importance: Undecided
         Status: New


** Tags: novnc

-- 
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to OpenStack Compute (nova).
https://bugs.launchpad.net/bugs/1800441

Title:
  Unexpected exception in API method: MessagingTimeout: Timed out
  waiting for a reply to message ID

Status in OpenStack Compute (nova):
  New

Bug description:
  Hello, I built all of rocky on one mainframe. The problem now is that
  I have no access to novnc. Everything else is okay.

  Some system versions and information:

  [root@all-in-one-202 ~]# cat /etc/centos-release
  CentOS Linux release 7.5.1804 (Core)

  [root@all-in-one-202 ~]# uname -a
  Linux all-in-one-202 3.10.0-862.11.6.el7.x86_64 #1 SMP Tue Aug 14 21:49:04 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux

  [root@all-in-one-202 ~]# rpm -qa | grep openstack-rocky
  centos-release-openstack-rocky-1-1.el7.centos.noarch

  [root@all-in-one-202 ~]# rpm -qa | grep memcache
  memcached-1.5.6-1.el7.x86_64
  python-memcached-1.58-1.el7.noarch

  [root@all-in-one-202 ~]# cat /etc/sysconfig/memcached
  PORT="11211"
  USER="memcached"
  MAXCONN="1024"
  CACHESIZE="64"
  OPTIONS=""

  [root@all-in-one-202 ~]# egrep -v "^#|^$" /etc/nova/nova.conf
  [DEFAULT]
  enabled_apis = osapi_compute,metadata
  transport_url = rabbit://openstack:Admin@10.100.26.202
  my_ip = 10.100.26.202
  use_neutron = true
  firewall_driver = nova.virt.firewall.NoopFirewallDriver
  enabled_apis = osapi_compute,metadata
  enable_network_quota=false
  [api]
  auth_strategy = keystone
  [api_database]
  connection = mysql+pymysql://nova:Admin@10.100.26.202/nova_api
  [barbican]
  [cache]
  memcached_servers=10.100.26.202:11211
  [cells]
  [cinder]
  [compute]
  [conductor]
  [console]
  [consoleauth]
  [cors]
  [database]
  connection = mysql+pymysql://nova:Admin@10.100.26.202/nova
  [devices]
  [ephemeral_storage_encryption]
  [filter_scheduler]
  [glance]
  api_servers = http://10.100.26.202:9292
  [guestfs]
  [healthcheck]
  [hyperv]
  [ironic]
  [key_manager]
  [keystone]
  [keystone_authtoken]
  auth_url = http://10.100.26.202:5000/v3
  memcached_servers = 10.100.26.202:11211
  auth_type = password
  project_domain_name = default
  user_domain_name = default
  project_name = service
  username = nova
  password = Admin
  [libvirt]
  virt_type = qemu
  [matchmaker_redis]
  [metrics]
  [mks]
  [neutron]
  url = http://10.100.26.202:9696
  auth_url = http://10.100.26.202:5000
  auth_type = password
  project_domain_name = default
  user_domain_name = default
  region_name = RegionOne
  project_name = service
  username = neutron
  password = Admin
  service_metadata_proxy = true
  metadata_proxy_shared_secret = Admin
  [notifications]
  [osapi_v21]
  [oslo_concurrency]
  lock_path = /var/lib/nova/tmp
  [oslo_messaging_amqp]
  [oslo_messaging_kafka]
  [oslo_messaging_notifications]
  [oslo_messaging_rabbit]
  [oslo_messaging_zmq]
  [oslo_middleware]
  [oslo_policy]
  [pci]
  [placement]
  region_name = RegionOne
  project_domain_name = Default
  project_name = service
  auth_type = password
  user_domain_name = Default
  auth_url = http://10.100.26.202:5000/v3
  username = placement
  password = Admin
  [placement_database]
  connection = mysql+pymysql://placement:Admin@10.100.26.202/placement
  [powervm]
  [profiler]
  [quota]
  instances=-1
  cores=-1
  ram=-1
  floating_ips=-1
  metadata_items=-1
  injected_files=-1
  injected_file_content_bytes=-1
  security_groups=-1
  security_group_rules=-1
  [rdp]
  [remote_debug]
  [scheduler]
  [serial_console]
  [service_user]
  [spice]
  [upgrade_levels]
  [vault]
  [vendordata_dynamic_auth]
  [vmware]
  [vnc]
  enabled = True
  server_listen = 10.100.26.202
  server_proxyclient_address = 10.100.26.202
  novncproxy_base_url = http://10.100.26.202:6080/vnc_auto.html
  [workarounds]
  [wsgi]
  [xenserver]
  [xvp]
  [zvm]

  [root@all-in-one-202 ~]# tail -200f /var/log/nova/nova-api.log

  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi [req-155d3560-8788-44be-ac3d-84b99baa1196 3e674e62b3cf490abbba13bab06449e2 63ff15cc51b7432a845633576b5183d6 - default default] Unexpected exception in API method: MessagingTimeout: Timed out waiting for a reply to message ID 991b3e95605c4ac6a33ba0a8ed722a4c
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi Traceback (most recent call last):
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/nova/api/openstack/wsgi.py", line 801, in wrapped
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     return f(*args, **kwargs)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/nova/api/validation/__init__.py", line 110, in wrapper
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     return func(*args, **kwargs)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/nova/api/openstack/compute/remote_consoles.py", line 52, in get_vnc_console
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     console_type)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/nova/compute/api.py", line 196, in wrapped
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     return function(self, context, instance, *args, **kwargs)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/nova/compute/api.py", line 186, in inner
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     return f(self, context, instance, *args, **kw)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/nova/compute/api.py", line 3750, in get_vnc_console
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     access_url=connect_info['access_url'])
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/nova/consoleauth/rpcapi.py", line 93, in authorize_console
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     return cctxt.call(ctxt, 'authorize_console', **msg_args)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/oslo_messaging/rpc/client.py", line 179, in call
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     retry=self.retry)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/oslo_messaging/transport.py", line 133, in _send
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     retry=retry)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 584, in send
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     call_monitor_timeout, retry=retry)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 573, in _send
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     call_monitor_timeout)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 459, in wait
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     message = self.waiters.get(msg_id, timeout=timeout)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 336, in get
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     'to message ID %s' % msg_id)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi MessagingTimeout: Timed out waiting for a reply to message ID 991b3e95605c4ac6a33ba0a8ed722a4c
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi
  2018-10-29 17:26:13.449 16284 INFO nova.api.openstack.wsgi [req-155d3560-8788-44be-ac3d-84b99baa1196 3e674e62b3cf490abbba13bab06449e2 63ff15cc51b7432a845633576b5183d6 - default default] HTTP exception thrown: Unexpected API Error. Please report this at http://bugs.launchpad.net/nova/ and attach the Nova API log if possible.
  <class 'oslo_messaging.exceptions.MessagingTimeout'>

To manage notifications about this bug go to:
https://bugs.launchpad.net/nova/+bug/1800441/+subscriptions
Follow ups

[Bug 1800441] Re: Unexpected exception in API method: MessagingTimeout: Timed out waiting for a reply to message ID
From: Matt Riedemann, 2018-11-01