yahoo-eng-team team mailing list archive

Thread
Date
[Bug 1420044] [NEW] Eventlet timeout when running test_l3_agent functional test

To: yahoo-eng-team@xxxxxxxxxxxxxxxxxxx
From: Terry Wilson <1420044@xxxxxxxxxxxxxxxxxx>
Date: Tue, 10 Feb 2015 00:00:15 -0000
Reply-to: Bug 1420044 <1420044@xxxxxxxxxxxxxxxxxx>
Sender: bounces@xxxxxxxxxxxxx
Public bug reported:

logstash: http://logs.openstack.org/84/153384/3/gate/gate-neutron-dsvm-
functional/7033858/console.html

There was an eventlet timeout during the above test run on a patch that
didn't change anything but some file locations. The timeout happened via
a eventlet.green.subprocess.Popen.communicate call. Log snip:


2015-02-09 22:07:05.960 | 2015-02-09 22:07:05.866 |     Traceback (most recent call last):
2015-02-09 22:07:05.961 | 2015-02-09 22:07:05.867 |       File "neutron/tests/functional/agent/test_l3_agent.py", line 520, in test_dvr_router_lifecycle_without_ha_with_snat_with_fips
2015-02-09 22:07:05.961 | 2015-02-09 22:07:05.868 |         self._dvr_router_lifecycle(enable_ha=False, enable_snat=True)
2015-02-09 22:07:05.962 | 2015-02-09 22:07:05.870 |       File "neutron/tests/functional/agent/test_l3_agent.py", line 571, in _dvr_router_lifecycle
2015-02-09 22:07:05.962 | 2015-02-09 22:07:05.871 |         self._delete_router(self.agent, router.router_id)
2015-02-09 22:07:05.962 | 2015-02-09 22:07:05.872 |       File "neutron/tests/functional/agent/test_l3_agent.py", line 116, in _delete_router
2015-02-09 22:07:05.963 | 2015-02-09 22:07:05.873 |         agent._router_removed(router_id)
2015-02-09 22:07:05.963 | 2015-02-09 22:07:05.875 |       File "neutron/agent/l3/agent.py", line 404, in _router_removed
2015-02-09 22:07:05.964 | 2015-02-09 22:07:05.876 |         self.process_router(ri)
2015-02-09 22:07:05.964 | 2015-02-09 22:07:05.877 |       File "neutron/common/utils.py", line 345, in call
2015-02-09 22:07:05.965 | 2015-02-09 22:07:05.879 |         self.logger(e)
2015-02-09 22:07:05.965 | 2015-02-09 22:07:05.880 |       File "/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 82, in __exit__
2015-02-09 22:07:05.966 | 2015-02-09 22:07:05.881 |         six.reraise(self.type_, self.value, self.tb)
2015-02-09 22:07:05.966 | 2015-02-09 22:07:05.883 |       File "neutron/common/utils.py", line 342, in call
2015-02-09 22:07:05.967 | 2015-02-09 22:07:05.884 |         return func(*args, **kwargs)
2015-02-09 22:07:05.967 | 2015-02-09 22:07:05.886 |       File "neutron/agent/l3/agent.py", line 599, in process_router
2015-02-09 22:07:05.968 | 2015-02-09 22:07:05.887 |         self._process_external(ri)
2015-02-09 22:07:05.968 | 2015-02-09 22:07:05.888 |       File "neutron/agent/l3/agent.py", line 564, in _process_external
2015-02-09 22:07:05.969 | 2015-02-09 22:07:05.889 |         self._process_external_gateway(ri)
2015-02-09 22:07:05.969 | 2015-02-09 22:07:05.890 |       File "neutron/agent/l3/agent.py", line 502, in _process_external_gateway
2015-02-09 22:07:05.970 | 2015-02-09 22:07:05.892 |         self.external_gateway_removed(ri, ri.ex_gw_port, interface_name)
2015-02-09 22:07:05.970 | 2015-02-09 22:07:05.893 |       File "neutron/agent/l3/agent.py", line 895, in external_gateway_removed
2015-02-09 22:07:05.971 | 2015-02-09 22:07:05.894 |         self.process_router_floating_ip_addresses(ri, interface_name)
2015-02-09 22:07:05.971 | 2015-02-09 22:07:05.895 |       File "neutron/agent/l3/agent.py", line 779, in process_router_floating_ip_addresses
2015-02-09 22:07:05.972 | 2015-02-09 22:07:05.897 |         self._remove_floating_ip(ri, device, ip_cidr)
2015-02-09 22:07:05.972 | 2015-02-09 22:07:05.898 |       File "neutron/agent/l3/agent.py", line 738, in _remove_floating_ip
2015-02-09 22:07:05.973 | 2015-02-09 22:07:05.899 |         self.floating_ip_removed_dist(ri, ip_cidr)
2015-02-09 22:07:05.973 | 2015-02-09 22:07:05.901 |       File "neutron/agent/l3/dvr.py", line 227, in floating_ip_removed_dist
2015-02-09 22:07:05.973 | 2015-02-09 22:07:05.902 |         ns_ip.del_veth(fip_2_rtr_name)
2015-02-09 22:07:05.974 | 2015-02-09 22:07:05.903 |       File "neutron/agent/linux/ip_lib.py", line 153, in del_veth
2015-02-09 22:07:05.975 | 2015-02-09 22:07:05.904 |         self._as_root('', 'link', ('del', name))
2015-02-09 22:07:05.975 | 2015-02-09 22:07:05.905 |       File "neutron/agent/linux/ip_lib.py", line 83, in _as_root
2015-02-09 22:07:05.976 | 2015-02-09 22:07:05.924 |         log_fail_as_error=self.log_fail_as_error)
2015-02-09 22:07:05.977 | 2015-02-09 22:07:05.925 |       File "neutron/agent/linux/ip_lib.py", line 95, in _execute
2015-02-09 22:07:05.979 | 2015-02-09 22:07:05.926 |         log_fail_as_error=log_fail_as_error)
2015-02-09 22:07:05.980 | 2015-02-09 22:07:05.928 |       File "neutron/agent/linux/utils.py", line 67, in execute
2015-02-09 22:07:05.981 | 2015-02-09 22:07:05.929 |         _stdout, _stderr = obj.communicate(process_input)
2015-02-09 22:07:05.981 | 2015-02-09 22:07:05.930 |       File "/usr/lib/python2.7/subprocess.py", line 799, in communicate
2015-02-09 22:07:05.982 | 2015-02-09 22:07:05.932 |         return self._communicate(input)
2015-02-09 22:07:05.983 | 2015-02-09 22:07:05.933 |       File "/usr/lib/python2.7/subprocess.py", line 1403, in _communicate
2015-02-09 22:07:05.983 | 2015-02-09 22:07:05.934 |         stdout, stderr = self._communicate_with_select(input)
2015-02-09 22:07:05.984 | 2015-02-09 22:07:05.936 |       File "/usr/lib/python2.7/subprocess.py", line 1504, in _communicate_with_select
2015-02-09 22:07:05.984 | 2015-02-09 22:07:05.937 |         rlist, wlist, xlist = select.select(read_set, write_set, [])
2015-02-09 22:07:05.985 | 2015-02-09 22:07:05.938 |       File "/usr/local/lib/python2.7/dist-packages/eventlet/green/select.py", line 83, in select
2015-02-09 22:07:05.986 | 2015-02-09 22:07:05.939 |         return hub.switch()
2015-02-09 22:07:05.986 | 2015-02-09 22:07:05.941 |       File "/usr/local/lib/python2.7/dist-packages/eventlet/hubs/hub.py", line 294, in switch
2015-02-09 22:07:05.987 | 2015-02-09 22:07:05.942 |         return self.greenlet.switch()
2015-02-09 22:07:05.988 | 2015-02-09 22:07:05.944 |       File "/usr/local/lib/python2.7/dist-packages/eventlet/hubs/hub.py", line 346, in run
2015-02-09 22:07:05.988 | 2015-02-09 22:07:05.946 |         self.wait(sleep_time)
2015-02-09 22:07:05.989 | 2015-02-09 22:07:05.948 |       File "/usr/local/lib/python2.7/dist-packages/eventlet/hubs/poll.py", line 85, in wait
2015-02-09 22:07:05.989 | 2015-02-09 22:07:05.949 |         presult = self.do_poll(seconds)
2015-02-09 22:07:05.990 | 2015-02-09 22:07:05.951 |       File "/usr/local/lib/python2.7/dist-packages/eventlet/hubs/epolls.py", line 62, in do_poll
2015-02-09 22:07:05.991 | 2015-02-09 22:07:05.952 |         return self.poll.poll(seconds)
2015-02-09 22:07:05.991 | 2015-02-09 22:07:05.953 |       File "/usr/local/lib/python2.7/dist-packages/fixtures/_fixtures/timeout.py", line 52, in signal_handler
2015-02-09 22:07:05.992 | 2015-02-09 22:07:05.955 |         raise TimeoutException()
2015-02-09 22:07:05.992 | 2015-02-09 22:07:05.957 |     TimeoutException

** Affects: neutron
     Importance: Undecided
         Status: New

-- 
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to neutron.
https://bugs.launchpad.net/bugs/1420044

Title:
  Eventlet timeout when running test_l3_agent functional test

Status in OpenStack Neutron (virtual network service):
  New

Bug description:
  logstash: http://logs.openstack.org/84/153384/3/gate/gate-neutron-
  dsvm-functional/7033858/console.html

  There was an eventlet timeout during the above test run on a patch
  that didn't change anything but some file locations. The timeout
  happened via a eventlet.green.subprocess.Popen.communicate call. Log
  snip:

  
  2015-02-09 22:07:05.960 | 2015-02-09 22:07:05.866 |     Traceback (most recent call last):
  2015-02-09 22:07:05.961 | 2015-02-09 22:07:05.867 |       File "neutron/tests/functional/agent/test_l3_agent.py", line 520, in test_dvr_router_lifecycle_without_ha_with_snat_with_fips
  2015-02-09 22:07:05.961 | 2015-02-09 22:07:05.868 |         self._dvr_router_lifecycle(enable_ha=False, enable_snat=True)
  2015-02-09 22:07:05.962 | 2015-02-09 22:07:05.870 |       File "neutron/tests/functional/agent/test_l3_agent.py", line 571, in _dvr_router_lifecycle
  2015-02-09 22:07:05.962 | 2015-02-09 22:07:05.871 |         self._delete_router(self.agent, router.router_id)
  2015-02-09 22:07:05.962 | 2015-02-09 22:07:05.872 |       File "neutron/tests/functional/agent/test_l3_agent.py", line 116, in _delete_router
  2015-02-09 22:07:05.963 | 2015-02-09 22:07:05.873 |         agent._router_removed(router_id)
  2015-02-09 22:07:05.963 | 2015-02-09 22:07:05.875 |       File "neutron/agent/l3/agent.py", line 404, in _router_removed
  2015-02-09 22:07:05.964 | 2015-02-09 22:07:05.876 |         self.process_router(ri)
  2015-02-09 22:07:05.964 | 2015-02-09 22:07:05.877 |       File "neutron/common/utils.py", line 345, in call
  2015-02-09 22:07:05.965 | 2015-02-09 22:07:05.879 |         self.logger(e)
  2015-02-09 22:07:05.965 | 2015-02-09 22:07:05.880 |       File "/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 82, in __exit__
  2015-02-09 22:07:05.966 | 2015-02-09 22:07:05.881 |         six.reraise(self.type_, self.value, self.tb)
  2015-02-09 22:07:05.966 | 2015-02-09 22:07:05.883 |       File "neutron/common/utils.py", line 342, in call
  2015-02-09 22:07:05.967 | 2015-02-09 22:07:05.884 |         return func(*args, **kwargs)
  2015-02-09 22:07:05.967 | 2015-02-09 22:07:05.886 |       File "neutron/agent/l3/agent.py", line 599, in process_router
  2015-02-09 22:07:05.968 | 2015-02-09 22:07:05.887 |         self._process_external(ri)
  2015-02-09 22:07:05.968 | 2015-02-09 22:07:05.888 |       File "neutron/agent/l3/agent.py", line 564, in _process_external
  2015-02-09 22:07:05.969 | 2015-02-09 22:07:05.889 |         self._process_external_gateway(ri)
  2015-02-09 22:07:05.969 | 2015-02-09 22:07:05.890 |       File "neutron/agent/l3/agent.py", line 502, in _process_external_gateway
  2015-02-09 22:07:05.970 | 2015-02-09 22:07:05.892 |         self.external_gateway_removed(ri, ri.ex_gw_port, interface_name)
  2015-02-09 22:07:05.970 | 2015-02-09 22:07:05.893 |       File "neutron/agent/l3/agent.py", line 895, in external_gateway_removed
  2015-02-09 22:07:05.971 | 2015-02-09 22:07:05.894 |         self.process_router_floating_ip_addresses(ri, interface_name)
  2015-02-09 22:07:05.971 | 2015-02-09 22:07:05.895 |       File "neutron/agent/l3/agent.py", line 779, in process_router_floating_ip_addresses
  2015-02-09 22:07:05.972 | 2015-02-09 22:07:05.897 |         self._remove_floating_ip(ri, device, ip_cidr)
  2015-02-09 22:07:05.972 | 2015-02-09 22:07:05.898 |       File "neutron/agent/l3/agent.py", line 738, in _remove_floating_ip
  2015-02-09 22:07:05.973 | 2015-02-09 22:07:05.899 |         self.floating_ip_removed_dist(ri, ip_cidr)
  2015-02-09 22:07:05.973 | 2015-02-09 22:07:05.901 |       File "neutron/agent/l3/dvr.py", line 227, in floating_ip_removed_dist
  2015-02-09 22:07:05.973 | 2015-02-09 22:07:05.902 |         ns_ip.del_veth(fip_2_rtr_name)
  2015-02-09 22:07:05.974 | 2015-02-09 22:07:05.903 |       File "neutron/agent/linux/ip_lib.py", line 153, in del_veth
  2015-02-09 22:07:05.975 | 2015-02-09 22:07:05.904 |         self._as_root('', 'link', ('del', name))
  2015-02-09 22:07:05.975 | 2015-02-09 22:07:05.905 |       File "neutron/agent/linux/ip_lib.py", line 83, in _as_root
  2015-02-09 22:07:05.976 | 2015-02-09 22:07:05.924 |         log_fail_as_error=self.log_fail_as_error)
  2015-02-09 22:07:05.977 | 2015-02-09 22:07:05.925 |       File "neutron/agent/linux/ip_lib.py", line 95, in _execute
  2015-02-09 22:07:05.979 | 2015-02-09 22:07:05.926 |         log_fail_as_error=log_fail_as_error)
  2015-02-09 22:07:05.980 | 2015-02-09 22:07:05.928 |       File "neutron/agent/linux/utils.py", line 67, in execute
  2015-02-09 22:07:05.981 | 2015-02-09 22:07:05.929 |         _stdout, _stderr = obj.communicate(process_input)
  2015-02-09 22:07:05.981 | 2015-02-09 22:07:05.930 |       File "/usr/lib/python2.7/subprocess.py", line 799, in communicate
  2015-02-09 22:07:05.982 | 2015-02-09 22:07:05.932 |         return self._communicate(input)
  2015-02-09 22:07:05.983 | 2015-02-09 22:07:05.933 |       File "/usr/lib/python2.7/subprocess.py", line 1403, in _communicate
  2015-02-09 22:07:05.983 | 2015-02-09 22:07:05.934 |         stdout, stderr = self._communicate_with_select(input)
  2015-02-09 22:07:05.984 | 2015-02-09 22:07:05.936 |       File "/usr/lib/python2.7/subprocess.py", line 1504, in _communicate_with_select
  2015-02-09 22:07:05.984 | 2015-02-09 22:07:05.937 |         rlist, wlist, xlist = select.select(read_set, write_set, [])
  2015-02-09 22:07:05.985 | 2015-02-09 22:07:05.938 |       File "/usr/local/lib/python2.7/dist-packages/eventlet/green/select.py", line 83, in select
  2015-02-09 22:07:05.986 | 2015-02-09 22:07:05.939 |         return hub.switch()
  2015-02-09 22:07:05.986 | 2015-02-09 22:07:05.941 |       File "/usr/local/lib/python2.7/dist-packages/eventlet/hubs/hub.py", line 294, in switch
  2015-02-09 22:07:05.987 | 2015-02-09 22:07:05.942 |         return self.greenlet.switch()
  2015-02-09 22:07:05.988 | 2015-02-09 22:07:05.944 |       File "/usr/local/lib/python2.7/dist-packages/eventlet/hubs/hub.py", line 346, in run
  2015-02-09 22:07:05.988 | 2015-02-09 22:07:05.946 |         self.wait(sleep_time)
  2015-02-09 22:07:05.989 | 2015-02-09 22:07:05.948 |       File "/usr/local/lib/python2.7/dist-packages/eventlet/hubs/poll.py", line 85, in wait
  2015-02-09 22:07:05.989 | 2015-02-09 22:07:05.949 |         presult = self.do_poll(seconds)
  2015-02-09 22:07:05.990 | 2015-02-09 22:07:05.951 |       File "/usr/local/lib/python2.7/dist-packages/eventlet/hubs/epolls.py", line 62, in do_poll
  2015-02-09 22:07:05.991 | 2015-02-09 22:07:05.952 |         return self.poll.poll(seconds)
  2015-02-09 22:07:05.991 | 2015-02-09 22:07:05.953 |       File "/usr/local/lib/python2.7/dist-packages/fixtures/_fixtures/timeout.py", line 52, in signal_handler
  2015-02-09 22:07:05.992 | 2015-02-09 22:07:05.955 |         raise TimeoutException()
  2015-02-09 22:07:05.992 | 2015-02-09 22:07:05.957 |     TimeoutException

To manage notifications about this bug go to:
https://bugs.launchpad.net/neutron/+bug/1420044/+subscriptions
Follow ups

[Bug 1420044] Re: Eventlet timeout when running test_l3_agent functional test
From: Launchpad Bug Tracker, 2015-06-06
[Bug 1420044] [NEW] Eventlet timeout when running test_l3_agent functional test
From: Terry Wilson, 2015-02-10
References

[Bug 1420044] [NEW] Eventlet timeout when running test_l3_agent functional test
From: Terry Wilson, 2015-02-10