yahoo-eng-team team mailing list archive

Thread
Date
[Bug 1800441] Re: Unexpected exception in API method: MessagingTimeout: Timed out waiting for a reply to message ID

To: yahoo-eng-team@xxxxxxxxxxxxxxxxxxx
From: Matt Riedemann <mriedem.os@xxxxxxxxx>
Date: Thu, 01 Nov 2018 15:55:54 -0000
Reply-to: Bug 1800441 <1800441@xxxxxxxxxxxxxxxxxx>
Sender: bounces@xxxxxxxxxxxxx
Well, is the nova-consoleauth service running? Looks like it's timing
out trying to communicate with the nova-consoleauth service, which you
might have not installed/configured due to some confusion over the
deprecation of that service in the Rocky release, see:

https://review.openstack.org/#/q/status:merged+project:openstack/nova+topic:bug/1798188

** Changed in: nova
       Status: New => Invalid

-- 
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to OpenStack Compute (nova).
https://bugs.launchpad.net/bugs/1800441

Title:
  Unexpected exception in API method: MessagingTimeout: Timed out
  waiting for a reply to message ID

Status in OpenStack Compute (nova):
  Invalid

Bug description:
  Hello, I built all of rocky on one mainframe. The problem now is that
  I have no access to novnc. Everything else is okay.

  Some system versions and information:

  [root@all-in-one-202 ~]# cat /etc/centos-release
  CentOS Linux release 7.5.1804 (Core)

  [root@all-in-one-202 ~]# uname -a
  Linux all-in-one-202 3.10.0-862.11.6.el7.x86_64 #1 SMP Tue Aug 14 21:49:04 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux

  [root@all-in-one-202 ~]# rpm -qa | grep openstack-rocky
  centos-release-openstack-rocky-1-1.el7.centos.noarch

  [root@all-in-one-202 ~]# rpm -qa | grep memcache
  memcached-1.5.6-1.el7.x86_64
  python-memcached-1.58-1.el7.noarch

  [root@all-in-one-202 ~]# cat /etc/sysconfig/memcached
  PORT="11211"
  USER="memcached"
  MAXCONN="1024"
  CACHESIZE="64"
  OPTIONS=""

  [root@all-in-one-202 ~]# egrep -v "^#|^$" /etc/nova/nova.conf
  [DEFAULT]
  enabled_apis = osapi_compute,metadata
  transport_url = rabbit://openstack:Admin@10.100.26.202
  my_ip = 10.100.26.202
  use_neutron = true
  firewall_driver = nova.virt.firewall.NoopFirewallDriver
  enabled_apis = osapi_compute,metadata
  enable_network_quota=false
  [api]
  auth_strategy = keystone
  [api_database]
  connection = mysql+pymysql://nova:Admin@10.100.26.202/nova_api
  [barbican]
  [cache]
  memcached_servers=10.100.26.202:11211
  [cells]
  [cinder]
  [compute]
  [conductor]
  [console]
  [consoleauth]
  [cors]
  [database]
  connection = mysql+pymysql://nova:Admin@10.100.26.202/nova
  [devices]
  [ephemeral_storage_encryption]
  [filter_scheduler]
  [glance]
  api_servers = http://10.100.26.202:9292
  [guestfs]
  [healthcheck]
  [hyperv]
  [ironic]
  [key_manager]
  [keystone]
  [keystone_authtoken]
  auth_url = http://10.100.26.202:5000/v3
  memcached_servers = 10.100.26.202:11211
  auth_type = password
  project_domain_name = default
  user_domain_name = default
  project_name = service
  username = nova
  password = Admin
  [libvirt]
  virt_type = qemu
  [matchmaker_redis]
  [metrics]
  [mks]
  [neutron]
  url = http://10.100.26.202:9696
  auth_url = http://10.100.26.202:5000
  auth_type = password
  project_domain_name = default
  user_domain_name = default
  region_name = RegionOne
  project_name = service
  username = neutron
  password = Admin
  service_metadata_proxy = true
  metadata_proxy_shared_secret = Admin
  [notifications]
  [osapi_v21]
  [oslo_concurrency]
  lock_path = /var/lib/nova/tmp
  [oslo_messaging_amqp]
  [oslo_messaging_kafka]
  [oslo_messaging_notifications]
  [oslo_messaging_rabbit]
  [oslo_messaging_zmq]
  [oslo_middleware]
  [oslo_policy]
  [pci]
  [placement]
  region_name = RegionOne
  project_domain_name = Default
  project_name = service
  auth_type = password
  user_domain_name = Default
  auth_url = http://10.100.26.202:5000/v3
  username = placement
  password = Admin
  [placement_database]
  connection = mysql+pymysql://placement:Admin@10.100.26.202/placement
  [powervm]
  [profiler]
  [quota]
  instances=-1
  cores=-1
  ram=-1
  floating_ips=-1
  metadata_items=-1
  injected_files=-1
  injected_file_content_bytes=-1
  security_groups=-1
  security_group_rules=-1
  [rdp]
  [remote_debug]
  [scheduler]
  [serial_console]
  [service_user]
  [spice]
  [upgrade_levels]
  [vault]
  [vendordata_dynamic_auth]
  [vmware]
  [vnc]
  enabled = True
  server_listen = 10.100.26.202
  server_proxyclient_address = 10.100.26.202
  novncproxy_base_url = http://10.100.26.202:6080/vnc_auto.html
  [workarounds]
  [wsgi]
  [xenserver]
  [xvp]
  [zvm]

  [root@all-in-one-202 ~]# tail -200f /var/log/nova/nova-api.log

  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi [req-155d3560-8788-44be-ac3d-84b99baa1196 3e674e62b3cf490abbba13bab06449e2 63ff15cc51b7432a845633576b5183d6 - default default] Unexpected exception in API method: MessagingTimeout: Timed out waiting for a reply to message ID 991b3e95605c4ac6a33ba0a8ed722a4c
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi Traceback (most recent call last):
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/nova/api/openstack/wsgi.py", line 801, in wrapped
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     return f(*args, **kwargs)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/nova/api/validation/__init__.py", line 110, in wrapper
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     return func(*args, **kwargs)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/nova/api/openstack/compute/remote_consoles.py", line 52, in get_vnc_console
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     console_type)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/nova/compute/api.py", line 196, in wrapped
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     return function(self, context, instance, *args, **kwargs)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/nova/compute/api.py", line 186, in inner
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     return f(self, context, instance, *args, **kw)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/nova/compute/api.py", line 3750, in get_vnc_console
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     access_url=connect_info['access_url'])
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/nova/consoleauth/rpcapi.py", line 93, in authorize_console
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     return cctxt.call(ctxt, 'authorize_console', **msg_args)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/oslo_messaging/rpc/client.py", line 179, in call
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     retry=self.retry)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/oslo_messaging/transport.py", line 133, in _send
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     retry=retry)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 584, in send
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     call_monitor_timeout, retry=retry)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 573, in _send
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     call_monitor_timeout)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 459, in wait
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     message = self.waiters.get(msg_id, timeout=timeout)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 336, in get
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi     'to message ID %s' % msg_id)
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi MessagingTimeout: Timed out waiting for a reply to message ID 991b3e95605c4ac6a33ba0a8ed722a4c
  2018-10-29 17:26:13.443 16284 ERROR nova.api.openstack.wsgi
  2018-10-29 17:26:13.449 16284 INFO nova.api.openstack.wsgi [req-155d3560-8788-44be-ac3d-84b99baa1196 3e674e62b3cf490abbba13bab06449e2 63ff15cc51b7432a845633576b5183d6 - default default] HTTP exception thrown: Unexpected API Error. Please report this at http://bugs.launchpad.net/nova/ and attach the Nova API log if possible.
  <class 'oslo_messaging.exceptions.MessagingTimeout'>

To manage notifications about this bug go to:
https://bugs.launchpad.net/nova/+bug/1800441/+subscriptions
References

[Bug 1800441] [NEW] Unexpected exception in API method: MessagingTimeout: Timed out waiting for a reply to message ID
From: chinassh1209, 2018-10-29