yahoo-eng-team team mailing list archive
-
yahoo-eng-team team
-
Mailing list archive
-
Message #55339
[Bug 1615403] [NEW] IPAM: intermittent subnet creation failure
Public bug reported:
2016-08-21 14:02:57.227 13823 ERROR neutron.db.db_base_plugin_v2 [req-7eb3586f-e5c8-440c-9252-108612807472 tempest-BulkNetworkOpsTest-2010636197 -] An exception occurred while creating the subnet:{'subnet': {'host_routes': <neutron_lib.constants.Sentinel object at 0x7f4cb9317f50>, 'prefixlen': <neutron_lib.constants.Sentinel object at 0x7f4cb9317f50>, 'name': '', 'enable_dhcp': True, u'network_id': u'e19504d7-6e97-4405-89f6-3a21b190c52f', 'tenant_id': u'654a05200b6f41eebbe3151e1c39e878', 'dns_nameservers': <neutron_lib.constants.Sentinel object at 0x7f4cb9317f50>, 'ipv6_ra_mode': <neutron_lib.constants.Sentinel object at 0x7f4cb9317f50>, 'allocation_pools': [IPRange('10.20.0.18', '10.20.0.30')], 'gateway_ip': '10.20.0.17', u'ip_version': 4, 'ipv6_address_mode': <neutron_lib.constants.Sentinel object at 0x7f4cb9317f50>, u'cidr': '10.20.0.16/28', 'network:tenant_id': u'654a05200b6f41eebbe3151e1c39e878', 'subnetpool_id': <neutron_lib.constants.Sentinel object at 0x7f4cb9317f50>, 'description': ''}}
2016-08-21 14:02:57.245 13823 DEBUG neutron.db.api [req-7eb3586f-e5c8-440c-9252-108612807472 tempest-BulkNetworkOpsTest-2010636197 -] Retry wrapper got retriable exception: Traceback (most recent call last):
File "/opt/stack/neutron/neutron/db/api.py", line 77, in wrapped
return f(*args, **kwargs)
File "/opt/stack/neutron/neutron/api/v2/base.py", line 496, in _create
objs = do_create(body, bulk=True)
File "/opt/stack/neutron/neutron/api/v2/base.py", line 492, in do_create
request.context, reservation.reservation_id)
File "/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 220, in __exit__
self.force_reraise()
File "/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise
six.reraise(self.type_, self.value, self.tb)
File "/opt/stack/neutron/neutron/api/v2/base.py", line 485, in do_create
return obj_creator(request.context, **kwargs)
File "/opt/stack/neutron/neutron/db/db_base_plugin_v2.py", line 442, in create_subnet_bulk
return self._create_bulk('subnet', context, subnets)
File "/opt/stack/neutron/neutron/db/db_base_plugin_v2.py", line 333, in _create_bulk
{'resource': resource, 'item': item})
File "/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 220, in __exit__
self.force_reraise()
File "/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise
six.reraise(self.type_, self.value, self.tb)
File "/opt/stack/neutron/neutron/db/db_base_plugin_v2.py", line 326, in _create_bulk
objects.append(obj_creator(context, item))
File "/opt/stack/vmware-nsx/vmware_nsx/plugins/nsx_v3/plugin.py", line 1004, in create_subnet
context, subnet)
File "/opt/stack/neutron/neutron/db/db_base_plugin_v2.py", line 723, in create_subnet
return self._create_subnet(context, subnet, subnetpool_id)
File "/opt/stack/neutron/neutron/db/db_base_plugin_v2.py", line 614, in _create_subnet
subnetpool_id)
File "/opt/stack/neutron/neutron/db/ipam_pluggable_backend.py", line 486, in allocate_subnet
subnet_request.subnet_id)
File "/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 220, in __exit__
self.force_reraise()
File "/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise
six.reraise(self.type_, self.value, self.tb)
File "/opt/stack/neutron/neutron/db/ipam_pluggable_backend.py", line 472, in allocate_subnet
subnet_request)
File "/opt/stack/neutron/neutron/db/ipam_backend_mixin.py", line 500, in _save_subnet
context.session.flush()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2019, in flush
self._flush(objects)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2137, in _flush
transaction.rollback(_capture_exception=True)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2101, in _flush
flush_context.execute()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 373, in execute
rec.execute(self)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 532, in execute
uow
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 174, in save_obj
mapper, table, insert)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 767, in _emit_insert_statements
execute(statement, multiparams)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 914, in execute
return meth(self, multiparams, params)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py", line 323, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1010, in _execute_clauseelement
compiled_sql, distilled_params
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1146, in _execute_context
context)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1337, in _handle_dbapi_exception
util.raise_from_cause(newraise, exc_info)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 202, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1139, in _execute_context
context)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 450, in do_execute
cursor.execute(statement, parameters)
File "/usr/local/lib/python2.7/dist-packages/pymysql/cursors.py", line 167, in execute
result = self._query(query)
File "/usr/local/lib/python2.7/dist-packages/pymysql/cursors.py", line 323, in _query
conn.query(q)
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 836, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 1020, in _read_query_result
result.read()
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 1303, in read
first_packet = self.connection._read_packet()
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 982, in _read_packet
packet.check_error()
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 394, in check_error
err.raise_mysql_exception(self._data)
File "/usr/local/lib/python2.7/dist-packages/pymysql/err.py", line 120, in raise_mysql_exception
_check_mysql_exception(errinfo)
File "/usr/local/lib/python2.7/dist-packages/pymysql/err.py", line 115, in _check_mysql_exception
raise InternalError(errno, errorvalue)
DBDeadlock: (pymysql.err.InternalError) (1213, u'Deadlock found when trying to get lock; try restarting transaction') [SQL: u'INSERT INTO ipamsubnets (id, neutron_subnet_id) VALUES (%(id)s, %(neutron_subnet_id)s)'] [parameters: {'neutron_subnet_id': '31d5aae4-e20a-4503-9f41-50ba18252d98', 'id': '11fcf179-dab7-4ea6-ba45-3c667e1208c1'}]
** Affects: neutron
Importance: Critical
Status: New
** Tags: l3-ipam-dhcp
** Changed in: neutron
Importance: Undecided => Critical
** Tags added: l3-ipam-dhcp
--
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to neutron.
https://bugs.launchpad.net/bugs/1615403
Title:
IPAM: intermittent subnet creation failure
Status in neutron:
New
Bug description:
2016-08-21 14:02:57.227 13823 ERROR neutron.db.db_base_plugin_v2 [req-7eb3586f-e5c8-440c-9252-108612807472 tempest-BulkNetworkOpsTest-2010636197 -] An exception occurred while creating the subnet:{'subnet': {'host_routes': <neutron_lib.constants.Sentinel object at 0x7f4cb9317f50>, 'prefixlen': <neutron_lib.constants.Sentinel object at 0x7f4cb9317f50>, 'name': '', 'enable_dhcp': True, u'network_id': u'e19504d7-6e97-4405-89f6-3a21b190c52f', 'tenant_id': u'654a05200b6f41eebbe3151e1c39e878', 'dns_nameservers': <neutron_lib.constants.Sentinel object at 0x7f4cb9317f50>, 'ipv6_ra_mode': <neutron_lib.constants.Sentinel object at 0x7f4cb9317f50>, 'allocation_pools': [IPRange('10.20.0.18', '10.20.0.30')], 'gateway_ip': '10.20.0.17', u'ip_version': 4, 'ipv6_address_mode': <neutron_lib.constants.Sentinel object at 0x7f4cb9317f50>, u'cidr': '10.20.0.16/28', 'network:tenant_id': u'654a05200b6f41eebbe3151e1c39e878', 'subnetpool_id': <neutron_lib.constants.Sentinel object at 0x7f4cb9317f50>, 'description': ''}}
2016-08-21 14:02:57.245 13823 DEBUG neutron.db.api [req-7eb3586f-e5c8-440c-9252-108612807472 tempest-BulkNetworkOpsTest-2010636197 -] Retry wrapper got retriable exception: Traceback (most recent call last):
File "/opt/stack/neutron/neutron/db/api.py", line 77, in wrapped
return f(*args, **kwargs)
File "/opt/stack/neutron/neutron/api/v2/base.py", line 496, in _create
objs = do_create(body, bulk=True)
File "/opt/stack/neutron/neutron/api/v2/base.py", line 492, in do_create
request.context, reservation.reservation_id)
File "/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 220, in __exit__
self.force_reraise()
File "/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise
six.reraise(self.type_, self.value, self.tb)
File "/opt/stack/neutron/neutron/api/v2/base.py", line 485, in do_create
return obj_creator(request.context, **kwargs)
File "/opt/stack/neutron/neutron/db/db_base_plugin_v2.py", line 442, in create_subnet_bulk
return self._create_bulk('subnet', context, subnets)
File "/opt/stack/neutron/neutron/db/db_base_plugin_v2.py", line 333, in _create_bulk
{'resource': resource, 'item': item})
File "/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 220, in __exit__
self.force_reraise()
File "/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise
six.reraise(self.type_, self.value, self.tb)
File "/opt/stack/neutron/neutron/db/db_base_plugin_v2.py", line 326, in _create_bulk
objects.append(obj_creator(context, item))
File "/opt/stack/vmware-nsx/vmware_nsx/plugins/nsx_v3/plugin.py", line 1004, in create_subnet
context, subnet)
File "/opt/stack/neutron/neutron/db/db_base_plugin_v2.py", line 723, in create_subnet
return self._create_subnet(context, subnet, subnetpool_id)
File "/opt/stack/neutron/neutron/db/db_base_plugin_v2.py", line 614, in _create_subnet
subnetpool_id)
File "/opt/stack/neutron/neutron/db/ipam_pluggable_backend.py", line 486, in allocate_subnet
subnet_request.subnet_id)
File "/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 220, in __exit__
self.force_reraise()
File "/usr/local/lib/python2.7/dist-packages/oslo_utils/excutils.py", line 196, in force_reraise
six.reraise(self.type_, self.value, self.tb)
File "/opt/stack/neutron/neutron/db/ipam_pluggable_backend.py", line 472, in allocate_subnet
subnet_request)
File "/opt/stack/neutron/neutron/db/ipam_backend_mixin.py", line 500, in _save_subnet
context.session.flush()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2019, in flush
self._flush(objects)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2137, in _flush
transaction.rollback(_capture_exception=True)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2101, in _flush
flush_context.execute()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 373, in execute
rec.execute(self)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 532, in execute
uow
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 174, in save_obj
mapper, table, insert)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 767, in _emit_insert_statements
execute(statement, multiparams)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 914, in execute
return meth(self, multiparams, params)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py", line 323, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1010, in _execute_clauseelement
compiled_sql, distilled_params
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1146, in _execute_context
context)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1337, in _handle_dbapi_exception
util.raise_from_cause(newraise, exc_info)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 202, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1139, in _execute_context
context)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 450, in do_execute
cursor.execute(statement, parameters)
File "/usr/local/lib/python2.7/dist-packages/pymysql/cursors.py", line 167, in execute
result = self._query(query)
File "/usr/local/lib/python2.7/dist-packages/pymysql/cursors.py", line 323, in _query
conn.query(q)
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 836, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 1020, in _read_query_result
result.read()
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 1303, in read
first_packet = self.connection._read_packet()
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 982, in _read_packet
packet.check_error()
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 394, in check_error
err.raise_mysql_exception(self._data)
File "/usr/local/lib/python2.7/dist-packages/pymysql/err.py", line 120, in raise_mysql_exception
_check_mysql_exception(errinfo)
File "/usr/local/lib/python2.7/dist-packages/pymysql/err.py", line 115, in _check_mysql_exception
raise InternalError(errno, errorvalue)
DBDeadlock: (pymysql.err.InternalError) (1213, u'Deadlock found when trying to get lock; try restarting transaction') [SQL: u'INSERT INTO ipamsubnets (id, neutron_subnet_id) VALUES (%(id)s, %(neutron_subnet_id)s)'] [parameters: {'neutron_subnet_id': '31d5aae4-e20a-4503-9f41-50ba18252d98', 'id': '11fcf179-dab7-4ea6-ba45-3c667e1208c1'}]
To manage notifications about this bug go to:
https://bugs.launchpad.net/neutron/+bug/1615403/+subscriptions
Follow ups
