yahoo-eng-team team mailing list archive

Thread
Date
[Bug 1481029] [NEW] router attachment dealock

To: yahoo-eng-team@xxxxxxxxxxxxxxxxxxx
From: Sergey Yudin <tsipa740@xxxxxxxxx>
Date: Mon, 03 Aug 2015 17:31:19 -0000
Reply-to: Bug 1481029 <1481029@xxxxxxxxxxxxxxxxxx>
Sender: bounces@xxxxxxxxxxxxx
Public bug reported:

When attaching router to network with active-active mysql galera i have
faced deadlock

 "_msg_id": "5f347f6c94224ef094ed17554f290159", "result": null, "ending": true}', 'oslo.version': '2.0'} with routing key reply_6a1ae2768c964d2095a16d205f0363f send /usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/impl_rabbit.py:400
2015-07-21T02:32:47.617456+00:00 err:  2015-07-21 02:32:47.407 10196 ERROR neutron.api.v2.resource [req-20a96c08-2f9a-432e-80a1-821c5a26812e ] add_router_intrface failed
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource Traceback (most recent call last):
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/neutron/api/v2/resource.py", line 83, in resource
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     result = method(request=request, **args)
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/neutron/api/v2/base.py", line 207, in _handle_action
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     return getattr(self._plugin, name)(*arg_list, **kwargs)
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/neutron/db/l3_dvr_db.py", line 289, in add_router_interfce
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     context, router, interface_info['subnet_id'], device_owner)
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/neutron/db/l3_db.py", line 597, in _add_interface_by_subet
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     'name': ''}}), [subnet], True
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 972, in create_port
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     result, mech_context = self._create_port_db(context, port)
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 955, in _create_por_db
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     binding = db.add_port_binding(session, result['id'])
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/db.py", line 128, in add_port_bindin
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     return record
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 470, in __exit__
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     self.rollback()
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     compat.reraise(exc_type, exc_value, exc_tb)
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 467, in __exit__
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     self.commit()
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 377, in commit
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     self._prepare_impl()
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 357, in _prepare_impl
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     self.session.flush()
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1919, in flush
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     self._flush(objects)
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2037, in _flush
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     transaction.rollback(_capture_exception=True)
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     compat.reraise(exc_type, exc_value, exc_tb)
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2001, in _flush
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     flush_context.execute()
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 372, in execute
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     rec.execute(self)
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 526, in execute
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     uow
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 65, in save_obj
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     mapper, table, insert)
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 570, in _emit_inser_statements
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     execute(statement, multiparams)
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 729, in execute
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     return meth(self, multiparams, params)
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py", line 321, in _execute_on_conection
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     return connection._execute_clauseelement(self, multiparams, params)
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 826, in _execute_clauselement
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     compiled_sql, distilled_params
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 958, in _execute_contex
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     context)
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/oslo_db/sqlalchemy/compat/handle_error.py", line 261, in_handle_dbapi_exception
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     e, statement, parameters, cursor, context)
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1156, in _handle_dbapi_xception
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     util.raise_from_cause(newraise, exc_info)
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 199, in raise_from_caus
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     reraise(type(exception), exception, tb=exc_tb)
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 951, in _execute_contex
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     context)
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 436, in do_execute
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     cursor.execute(statement, parameters)
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/MySQLdb/cursors.py", line 205, in execute
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     self.errorhandler(self, exc, value)
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     raise errorclass, errorvalue
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource DBDeadlock: (OperationalError) (1213, 'Deadlock found when trying to get lock; try restarting trasaction') 'INSERT INTO ml2_port_bindings (port_id, host, vnic_type, profile, vif_type, vif_details) VALUES (%s, %s, %s, %s, %s, %s)' ('a972d0a4-3c6b-410c-8d1-069e2eb2bca7', '', 'normal', '', 'unbound', '')
2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource
2015-07-21T02:32:47.641616+00:00 info:  2015-07-21 02:32:47.637 10196 INFO neutron.wsgi [req-20a96c08-2f9a-432e-80a1-821c5a26812e ] 10.109.52.3 - - [21/Jul/215 02:32:47] "PUT /v2.0/routers/0071d2dc-7bfa-4a49-9508-8e3b1fbdb3fe/add_router_interface.json HTTP/1.1" 500 378 1.232624

** Affects: neutron
     Importance: Undecided
         Status: New

-- 
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to neutron.
https://bugs.launchpad.net/bugs/1481029

Title:
  router attachment dealock

Status in neutron:
  New

Bug description:
  When attaching router to network with active-active mysql galera i
  have faced deadlock

   "_msg_id": "5f347f6c94224ef094ed17554f290159", "result": null, "ending": true}', 'oslo.version': '2.0'} with routing key reply_6a1ae2768c964d2095a16d205f0363f send /usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/impl_rabbit.py:400
  2015-07-21T02:32:47.617456+00:00 err:  2015-07-21 02:32:47.407 10196 ERROR neutron.api.v2.resource [req-20a96c08-2f9a-432e-80a1-821c5a26812e ] add_router_intrface failed
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource Traceback (most recent call last):
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/neutron/api/v2/resource.py", line 83, in resource
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     result = method(request=request, **args)
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/neutron/api/v2/base.py", line 207, in _handle_action
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     return getattr(self._plugin, name)(*arg_list, **kwargs)
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/neutron/db/l3_dvr_db.py", line 289, in add_router_interfce
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     context, router, interface_info['subnet_id'], device_owner)
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/neutron/db/l3_db.py", line 597, in _add_interface_by_subet
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     'name': ''}}), [subnet], True
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 972, in create_port
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     result, mech_context = self._create_port_db(context, port)
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 955, in _create_por_db
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     binding = db.add_port_binding(session, result['id'])
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/db.py", line 128, in add_port_bindin
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     return record
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 470, in __exit__
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     self.rollback()
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     compat.reraise(exc_type, exc_value, exc_tb)
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 467, in __exit__
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     self.commit()
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 377, in commit
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     self._prepare_impl()
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 357, in _prepare_impl
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     self.session.flush()
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1919, in flush
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     self._flush(objects)
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2037, in _flush
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     transaction.rollback(_capture_exception=True)
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     compat.reraise(exc_type, exc_value, exc_tb)
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2001, in _flush
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     flush_context.execute()
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 372, in execute
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     rec.execute(self)
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 526, in execute
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     uow
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 65, in save_obj
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     mapper, table, insert)
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 570, in _emit_inser_statements
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     execute(statement, multiparams)
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 729, in execute
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     return meth(self, multiparams, params)
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py", line 321, in _execute_on_conection
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     return connection._execute_clauseelement(self, multiparams, params)
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 826, in _execute_clauselement
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     compiled_sql, distilled_params
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 958, in _execute_contex
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     context)
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/oslo_db/sqlalchemy/compat/handle_error.py", line 261, in_handle_dbapi_exception
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     e, statement, parameters, cursor, context)
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1156, in _handle_dbapi_xception
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     util.raise_from_cause(newraise, exc_info)
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 199, in raise_from_caus
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     reraise(type(exception), exception, tb=exc_tb)
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 951, in _execute_contex
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     context)
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 436, in do_execute
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     cursor.execute(statement, parameters)
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/MySQLdb/cursors.py", line 205, in execute
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     self.errorhandler(self, exc, value)
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource   File "/usr/lib/python2.7/dist-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource     raise errorclass, errorvalue
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource DBDeadlock: (OperationalError) (1213, 'Deadlock found when trying to get lock; try restarting trasaction') 'INSERT INTO ml2_port_bindings (port_id, host, vnic_type, profile, vif_type, vif_details) VALUES (%s, %s, %s, %s, %s, %s)' ('a972d0a4-3c6b-410c-8d1-069e2eb2bca7', '', 'normal', '', 'unbound', '')
  2015-07-21 02:32:47.407 10196 TRACE neutron.api.v2.resource
  2015-07-21T02:32:47.641616+00:00 info:  2015-07-21 02:32:47.637 10196 INFO neutron.wsgi [req-20a96c08-2f9a-432e-80a1-821c5a26812e ] 10.109.52.3 - - [21/Jul/215 02:32:47] "PUT /v2.0/routers/0071d2dc-7bfa-4a49-9508-8e3b1fbdb3fe/add_router_interface.json HTTP/1.1" 500 378 1.232624

To manage notifications about this bug go to:
https://bugs.launchpad.net/neutron/+bug/1481029/+subscriptions