yahoo-eng-team team mailing list archive

Thread
Date
[Bug 1908408] [NEW] Need to retry vmware instance creation

To: yahoo-eng-team@xxxxxxxxxxxxxxxxxxx
From: Adit Sarfaty <1908408@xxxxxxxxxxxxxxxxxx>
Date: Wed, 16 Dec 2020 13:35:50 -0000
Reply-to: Bug 1908408 <1908408@xxxxxxxxxxxxxxxxxx>
Sender: bounces@xxxxxxxxxxxxx
Public bug reported:

In some cases vm boot fails with this trace:

^[[01;31mERROR nova.compute.manager [^[[01;36mNone req-93f69841-12ab-42c5-b56a-83e964f4a374 ^[[00;36mtempest-TestNetworkBasicOps-721884738 tempest-TestNetworkBasicOps-721884738^[[01;31m] ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[01;31mInstance failed too
 spawn^[[00m: oslo_vmware.exceptions.VimFaultException: Network interface 'VirtualE1000' uses network 'tempest-network-smoke--802933861_b25e6...71f8b (nsx.LogicalSwitch:6b0a691d-794f-42f7-b648-d67e8a10c0eb)', which is not accessible.
Faults: ['CannotAccessNetwork']
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00mTraceback (most recent call last):
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/opt/stack/nova/nova/compute/manager.py", line 2663, in _build_resources
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    yield resources
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/opt/stack/nova/nova/compute/manager.py", line 2437, in _build_and_run_instance
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    block_device_info=block_device_info)
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 544, in spawn
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    admin_password, network_info, block_device_info)
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    metadata)
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 304, in build_virtual_machine
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    config_spec, self._root_resource_pool)
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1392, in create_vm
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    {'ostype': config_spec.guestId})
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/usr/local/lib/python3.6/dist-packages/oslo_utils/excutils.py", line 220, in __exit__
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    self.force_reraise()
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/usr/local/lib/python3.6/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    six.reraise(self.type_, self.value, self.tb)
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/usr/local/lib/python3.6/dist-packages/six.py", line 693, in reraise
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    raise value
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1377, in create_vm
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    task_info = session._wait_for_task(vm_create_task)
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 771, in _wait_for_task
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    return self.wait_for_task(task_ref)
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/usr/local/lib/python3.6/dist-packages/oslo_vmware/api.py", line 399, in wait_for_task
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    return evt.wait()
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/usr/local/lib/python3.6/dist-packages/eventlet/event.py", line 125, in wait
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    result = hub.switch()
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/usr/local/lib/python3.6/dist-packages/eventlet/hubs/hub.py", line 298, in switch
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    return self.greenlet.switch()
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/usr/local/lib/python3.6/dist-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    self.f(*self.args, **self.kw)
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/usr/local/lib/python3.6/dist-packages/oslo_vmware/api.py", line 458, in _poll_task
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    raise task_ex
ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00moslo_vmware.exceptions.VimFaultException: Network interface 'VirtualE1000' uses network 'tempest-network-smoke--802933861_b25e6...71f8b (nsx.LogicalSwitch:6b0a691d-794f-42f7-b648-d66
7e8a10c0eb)', which is not accessible.


Retry will solve the problem here, and the logical switch creation will be completed.

** Affects: nova
     Importance: Undecided
         Status: New

-- 
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to OpenStack Compute (nova).
https://bugs.launchpad.net/bugs/1908408

Title:
  Need to retry vmware instance creation

Status in OpenStack Compute (nova):
  New

Bug description:
  In some cases vm boot fails with this trace:

  ^[[01;31mERROR nova.compute.manager [^[[01;36mNone req-93f69841-12ab-42c5-b56a-83e964f4a374 ^[[00;36mtempest-TestNetworkBasicOps-721884738 tempest-TestNetworkBasicOps-721884738^[[01;31m] ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[01;31mInstance failed too
   spawn^[[00m: oslo_vmware.exceptions.VimFaultException: Network interface 'VirtualE1000' uses network 'tempest-network-smoke--802933861_b25e6...71f8b (nsx.LogicalSwitch:6b0a691d-794f-42f7-b648-d67e8a10c0eb)', which is not accessible.
  Faults: ['CannotAccessNetwork']
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00mTraceback (most recent call last):
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/opt/stack/nova/nova/compute/manager.py", line 2663, in _build_resources
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    yield resources
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/opt/stack/nova/nova/compute/manager.py", line 2437, in _build_and_run_instance
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    block_device_info=block_device_info)
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 544, in spawn
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    admin_password, network_info, block_device_info)
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    metadata)
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 304, in build_virtual_machine
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    config_spec, self._root_resource_pool)
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1392, in create_vm
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    {'ostype': config_spec.guestId})
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/usr/local/lib/python3.6/dist-packages/oslo_utils/excutils.py", line 220, in __exit__
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    self.force_reraise()
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/usr/local/lib/python3.6/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    six.reraise(self.type_, self.value, self.tb)
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/usr/local/lib/python3.6/dist-packages/six.py", line 693, in reraise
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    raise value
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1377, in create_vm
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    task_info = session._wait_for_task(vm_create_task)
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 771, in _wait_for_task
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    return self.wait_for_task(task_ref)
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/usr/local/lib/python3.6/dist-packages/oslo_vmware/api.py", line 399, in wait_for_task
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    return evt.wait()
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/usr/local/lib/python3.6/dist-packages/eventlet/event.py", line 125, in wait
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    result = hub.switch()
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/usr/local/lib/python3.6/dist-packages/eventlet/hubs/hub.py", line 298, in switch
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    return self.greenlet.switch()
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/usr/local/lib/python3.6/dist-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    self.f(*self.args, **self.kw)
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m  File "/usr/local/lib/python3.6/dist-packages/oslo_vmware/api.py", line 458, in _poll_task
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00m    raise task_ex
  ERROR nova.compute.manager ^[[01;35m[instance: 26d7da79-438a-41e2-bb57-8019f90f3fa2] ^[[00moslo_vmware.exceptions.VimFaultException: Network interface 'VirtualE1000' uses network 'tempest-network-smoke--802933861_b25e6...71f8b (nsx.LogicalSwitch:6b0a691d-794f-42f7-b648-d66
  7e8a10c0eb)', which is not accessible.

  
  Retry will solve the problem here, and the logical switch creation will be completed.

To manage notifications about this bug go to:
https://bugs.launchpad.net/nova/+bug/1908408/+subscriptions
Follow ups

[Bug 1908408] Re: Need to retry vmware instance creation
From: OpenStack Infra, 2021-08-24