← Back to team overview

yahoo-eng-team team mailing list archive

[Bug 1479818] [NEW] "Deadlock found" and "DuplicateEntry" errors when trying to attach a neutron L3 HA router to an internal network

 

Public bug reported:

On a setup with 2 neutron nodes, L3 agent in HA (VRRP), we get errors
while attaching a router to an internal network. How to replicate:

* create the external network (vlan)
* create an internal network (vxlan)
* create a router
* attach the router to the external network
* attach the router to the internal network

After attaching the router to the internal network, the following
happens:

a) some router _and_ dhcp ports of the external networks go from ACTIVE to BUILD status
b) dhcp ports of the internal networks go from ACTIVE to BUILD
c) two type of errors are found on the neutron log file:
   - a **Deadlock found when trying to get a lock** error, on one of the network nodes
   - multiple **Duplicate entry ... for key PRIMARY** on both network nodes

How we replicated the issue

    root@kenny:~# neutron  net-create uzh-only \
      --shared --provider:network_type vlan \
      --provider:segmentation_id 842 \
      --provider:physical_network vlannet \
      --router:external 

    root@kenny:~# neutron  subnet-create uzh-only \
        --name vlan842 \
        172.23.0.0/16 \
        --gateway 172.23.0.1 \
        --allocation-pool start=172.23.2.0,end=172.23.200.255 \
        --enable-dhcp \
        --dns-nameservers 130.60.128.3 130.60.64.51 \
        --   --host-routes type=dict list=true \
        destination=130.60.206.69/32,nexthop=172.23.0.5

    root@kenny:~# neutron net-create test 
    root@kenny:~# neutron subnet-create test --name subtest 10.0.0.0/24 
    root@kenny:~# neutron router-create test
    root@kenny:~# neutron router-gateway-set test uzh-only

    root@kenny:~# neutron port-list -c id -c fixed_ips -c status
    +--------------------------------------+--------------------------------------------------------------------------------------+--------+-----------------------------+
    | id                                   | fixed_ips                                                                            | status | device_owner                |
    +--------------------------------------+--------------------------------------------------------------------------------------+--------+-----------------------------+
    | 1463b78c-f640-4e41-8831-927679be380b | {"subnet_id": "e5e43019-58e4-4203-a51b-8e0dce7ca829", "ip_address": "10.0.0.3"}      | ACTIVE | network:dhcp                |
    | 1e7a0ac2-268c-48b7-9c70-e7e03126d9ed | {"subnet_id": "178f49ec-383e-4516-b6c3-814e7f459497", "ip_address": "169.254.192.1"} | ACTIVE | network:router_ha_interface |
    | 234ddb9b-16cc-48f2-b646-fee5f6d5def2 | {"subnet_id": "88a38e12-9402-418f-b0b7-1deeb843edf2", "ip_address": "172.23.2.2"}    | BUILD  | network:router_gateway      |
    | 3e755174-14b3-48e9-af84-489cce8985be | {"subnet_id": "e5e43019-58e4-4203-a51b-8e0dce7ca829", "ip_address": "10.0.0.2"}      | ACTIVE | network:dhcp                |
    | 4df3d3ab-1967-4bf1-bc3f-31ec1addd8d1 | {"subnet_id": "e5e43019-58e4-4203-a51b-8e0dce7ca829", "ip_address": "10.0.0.1"}      | BUILD  | network:router_interface    |
    | 5a2e6cbd-4c69-47e0-bfa6-92fe6242aba4 | {"subnet_id": "88a38e12-9402-418f-b0b7-1deeb843edf2", "ip_address": "172.23.2.0"}    | ACTIVE | network:dhcp                |
    | 881882f0-4ad7-4294-bec4-93f869cb40d8 | {"subnet_id": "178f49ec-383e-4516-b6c3-814e7f459497", "ip_address": "169.254.192.2"} | ACTIVE | network:router_ha_interface |
    | ac0e1761-335e-431f-ae89-a2703f2bafa5 | {"subnet_id": "88a38e12-9402-418f-b0b7-1deeb843edf2", "ip_address": "172.23.2.1"}    | ACTIVE | network:dhcp                |
    +--------------------------------------+--------------------------------------------------------------------------------------+--------+-----------------------------+

    root@kenny:~# neutron router-interface-add test subtest

    root@kenny:~# neutron port-list -c id -c fixed_ips -c status

    root@kenny:~# neutron port-list -c id -c fixed_ips -c status -c device_owner
    +--------------------------------------+--------------------------------------------------------------------------------------+--------+-----------------------------+
    | id                                   | fixed_ips                                                                            | status | device_owner                |
    +--------------------------------------+--------------------------------------------------------------------------------------+--------+-----------------------------+
    | 1463b78c-f640-4e41-8831-927679be380b | {"subnet_id": "e5e43019-58e4-4203-a51b-8e0dce7ca829", "ip_address": "10.0.0.3"}      | BUILD  | network:dhcp                |
    | 1e7a0ac2-268c-48b7-9c70-e7e03126d9ed | {"subnet_id": "178f49ec-383e-4516-b6c3-814e7f459497", "ip_address": "169.254.192.1"} | ACTIVE | network:router_ha_interface |
    | 234ddb9b-16cc-48f2-b646-fee5f6d5def2 | {"subnet_id": "88a38e12-9402-418f-b0b7-1deeb843edf2", "ip_address": "172.23.2.2"}    | BUILD  | network:router_gateway      |
    | 3e755174-14b3-48e9-af84-489cce8985be | {"subnet_id": "e5e43019-58e4-4203-a51b-8e0dce7ca829", "ip_address": "10.0.0.2"}      | ACTIVE | network:dhcp                |
    | 4df3d3ab-1967-4bf1-bc3f-31ec1addd8d1 | {"subnet_id": "e5e43019-58e4-4203-a51b-8e0dce7ca829", "ip_address": "10.0.0.1"}      | BUILD  | network:router_interface    |
    | 5a2e6cbd-4c69-47e0-bfa6-92fe6242aba4 | {"subnet_id": "88a38e12-9402-418f-b0b7-1deeb843edf2", "ip_address": "172.23.2.0"}    | ACTIVE | network:dhcp                |
    | 881882f0-4ad7-4294-bec4-93f869cb40d8 | {"subnet_id": "178f49ec-383e-4516-b6c3-814e7f459497", "ip_address": "169.254.192.2"} | BUILD  | network:router_ha_interface |
    | ac0e1761-335e-431f-ae89-a2703f2bafa5 | {"subnet_id": "88a38e12-9402-418f-b0b7-1deeb843edf2", "ip_address": "172.23.2.1"}    | BUILD  | network:dhcp                |
    +--------------------------------------+--------------------------------------------------------------------------------------+--------+-----------------------------+

excerpt from the logs on controller node A (only WARNING, ERROR and
TRACE)

    2015-07-30 15:50:59.322 251400 WARNING neutron.api.rpc.handlers.dhcp_rpc [req-203d6d37-4643-46e1-baa4-eb58c5f45750 ] Action create_port for network 9f7e9af9-72fc-44f1-89e3-be5cdf3d5bc5 could not complete successfully: (OperationalError) (1213, 'Deadlock found when trying to get lock; try restarting transaction') 'INSERT INTO ml2_port_binding_levels (port_id, host, level, driver, segment_id) VALUES (%s, %s, %s, %s, %s)' ('3e755174-14b3-48e9-af84-489cce8985be', 'cloud-l7-41', 0, 'openvswitch', '9a00e2d3-d842-4517-bec2-e3296ae711a5')
    2015-07-30 15:51:46.687 228137 ERROR neutron.plugins.openvswitch.agent.ovs_neutron_agent [req-0c92ed7e-f3c8-4d56-92dd-6ee597d25143 ] Error while processing VIF ports
    2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent Traceback (most recent call last):
    2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py", line 1586, in rpc_loop
    2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     ovs_restarted)
    2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py", line 1362, in process_network_ports
    2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     devices_added_updated, ovs_restarted)
    2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py", line 1267, in treat_devices_added_or_updated
    2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     self.context, device, self.agent_id, cfg.CONF.host)
    2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/agent/rpc.py", line 128, in update_device_up
    2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     agent_id=agent_id, host=host)
    2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/client.py", line 156, in call
    2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     retry=self.retry)
    2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/oslo_messaging/transport.py", line 90, in _send
    2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     timeout=timeout, retry=retry)
    2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py", line 350, in send
    2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     retry=retry)
    2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py", line 341, in _send
    2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     raise result
    2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent RemoteError: Remote error: DBDuplicateEntry (IntegrityError) (1062, "Duplicate entry '234ddb9b-16cc-48f2-b646-fee5f6d5def2' for key 'PRIMARY'") 'UPDATE ports SET status=%s WHERE ports.id = %s' ('ACTIVE', '234ddb9b-16cc-48f2-b646-fee5f6d5def2')
    2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent [u'Traceback (most recent call last):\n', u'  File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 142, in _dispatch_and_reply\n    executor_callback))\n', u'  File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 186, in _dispatch\n    executor_callback)\n', u'  File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 130, in _do_dispatch\n    result = func(ctxt, **new_args)\n', u'  File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/rpc.py", line 186, in update_device_up\n    host)\n', u'  File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 1373, in update_port_status\n    original_port[\'network_id\'])\n', u'  File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 658, in get_network\n    result = super(Ml2Plugin, self).get_network(context, id, None)\n', u'  File "/usr/lib/python2.7/dist-packages/neutron/db/db_base_plugin_v2.py", line 1007, in get_network\n    network = self._get_network(context, id)\n', u'  File "/usr/lib/python2.7/dist-packages/neutron/db/db_base_plugin_v2.py", line 91, in _get_network\n    network = self._get_by_id(context, models_v2.Network, id)\n', u'  File "/usr/lib/python2.7/dist-packages/neutron/db/common_db_mixin.py", line 130, in _get_by_id\n    return query.filter(model.id == id).one()\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2395, in one\n    ret = list(self)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2437, in __iter__\n    self.session._autoflush()\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1198, in _autoflush\n    self.flush()\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1919, in flush\n    self._flush(objects)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2037, in _flush\n    transaction.rollback(_capture_exception=True)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__\n    compat.reraise(exc_type, exc_value, exc_tb)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2001, in _flush\n    flush_context.execute()\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 372, in execute\n    rec.execute(self)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 526, in execute\n    uow\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 60, in save_obj\n    mapper, table, update)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 518, in _emit_update_statements\n    execute(statement, params)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 729, in execute\n    return meth(self, multiparams, params)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py", line 322, in _execute_on_connection\n    return connection._execute_clauseelement(self, multiparams, params)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 826, in _execute_clauseelement\n    compiled_sql, distilled_params\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 958, in _execute_context\n    context)\n', u'  File "/usr/lib/python2.7/dist-packages/oslo_db/sqlalchemy/compat/handle_error.py", line 261, in _handle_dbapi_exception\n    e, statement, parameters, cursor, context)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1155, in _handle_dbapi_exception\n    util.raise_from_cause(newraise, exc_info)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause\n    reraise(type(exception), exception, tb=exc_tb)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 951, in _execute_context\n    context)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 436, in do_execute\n    cursor.execute(statement, parameters)\n', u'  File "/usr/lib/python2.7/dist-packages/MySQLdb/cursors.py", line 174, in execute\n    self.errorhandler(self, exc, value)\n', u'  File "/usr/lib/python2.7/dist-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler\n    raise errorclass, errorvalue\n', u'DBDuplicateEntry: (IntegrityError) (1062, "Duplicate entry \'234ddb9b-16cc-48f2-b646-fee5f6d5def2\' for key \'PRIMARY\'") \'UPDATE ports SET status=%s WHERE ports.id = %s\' (\'ACTIVE\', \'234ddb9b-16cc-48f2-b646-fee5f6d5def2\')\n'].

excerpt from the logs on controller node B (only WARNING, ERROR and
TRACE)

    2015-07-30 15:51:46.680 55894 ERROR oslo_messaging.rpc.dispatcher [req-0c92ed7e-f3c8-4d56-92dd-6ee597d25143 ] Exception during message handling: (IntegrityError) (1062, "Duplicate entry '234ddb9b-16cc-48f2-b646-fee5f6d5def2' for key 'PRIMARY'") 'UPDATE ports SET status=%s WHERE ports.id = %s' ('ACTIVE', '234ddb9b-16cc-48f2-b646-fee5f6d5def2')
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher Traceback (most recent call last):
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 142, in _dispatch_and_reply
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     executor_callback))
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 186, in _dispatch
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     executor_callback)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 130, in _do_dispatch
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     result = func(ctxt, **new_args)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/rpc.py", line 186, in update_device_up
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     host)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 1373, in update_port_status
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     original_port['network_id'])
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 658, in get_network
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     result = super(Ml2Plugin, self).get_network(context, id, None)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/neutron/db/db_base_plugin_v2.py", line 1007, in get_network
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     network = self._get_network(context, id)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/neutron/db/db_base_plugin_v2.py", line 91, in _get_network
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     network = self._get_by_id(context, models_v2.Network, id)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/neutron/db/common_db_mixin.py", line 130, in _get_by_id
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     return query.filter(model.id == id).one()
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2395, in one
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     ret = list(self)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2437, in __iter__
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     self.session._autoflush()
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1198, in _autoflush
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     self.flush()
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1919, in flush
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     self._flush(objects)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2037, in _flush
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     transaction.rollback(_capture_exception=True)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     compat.reraise(exc_type, exc_value, exc_tb)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2001, in _flush
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     flush_context.execute()
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 372, in execute
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     rec.execute(self)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 526, in execute
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     uow
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 60, in save_obj
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     mapper, table, update)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 518, in _emit_update_statements
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     execute(statement, params)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 729, in execute
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     return meth(self, multiparams, params)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py", line 322, in _execute_on_connection
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     return connection._execute_clauseelement(self, multiparams, params)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 826, in _execute_clauseelement
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     compiled_sql, distilled_params
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 958, in _execute_context
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     context)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/oslo_db/sqlalchemy/compat/handle_error.py", line 261, in _handle_dbapi_exception
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     e, statement, parameters, cursor, context)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1155, in _handle_dbapi_exception
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     util.raise_from_cause(newraise, exc_info)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     reraise(type(exception), exception, tb=exc_tb)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 951, in _execute_context
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     context)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 436, in do_execute
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     cursor.execute(statement, parameters)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/MySQLdb/cursors.py", line 174, in execute
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     self.errorhandler(self, exc, value)
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     raise errorclass, errorvalue
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher DBDuplicateEntry: (IntegrityError) (1062, "Duplicate entry '234ddb9b-16cc-48f2-b646-fee5f6d5def2' for key 'PRIMARY'") 'UPDATE ports SET status=%s WHERE ports.id = %s' ('ACTIVE', '234ddb9b-16cc-48f2-b646-fee5f6d5def2')
    2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher 
    2015-07-30 15:51:46.682 55894 ERROR oslo_messaging._drivers.common [req-0c92ed7e-f3c8-4d56-92dd-6ee597d25143 ] Returning exception (IntegrityError) (1062, "Duplicate entry '234ddb9b-16cc-48f2-b646-fee5f6d5def2' for key 'PRIMARY'") 'UPDATE ports SET status=%s WHERE ports.id = %s' ('ACTIVE', '234ddb9b-16cc-48f2-b646-fee5f6d5def2') to caller
    2015-07-30 15:51:46.682 55894 ERROR oslo_messaging._drivers.common [req-0c92ed7e-f3c8-4d56-92dd-6ee597d25143 ] ['Traceback (most recent call last):\n', '  File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 142, in _dispatch_and_reply\n    executor_callback))\n', '  File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 186, in _dispatch\n    executor_callback)\n', '  File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 130, in _do_dispatch\n    result = func(ctxt, **new_args)\n', '  File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/rpc.py", line 186, in update_device_up\n    host)\n', '  File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 1373, in update_port_status\n    original_port[\'network_id\'])\n', '  File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 658, in get_network\n    result = super(Ml2Plugin, self).get_network(context, id, None)\n', '  File "/usr/lib/python2.7/dist-packages/neutron/db/db_base_plugin_v2.py", line 1007, in get_network\n    network = self._get_network(context, id)\n', '  File "/usr/lib/python2.7/dist-packages/neutron/db/db_base_plugin_v2.py", line 91, in _get_network\n    network = self._get_by_id(context, models_v2.Network, id)\n', '  File "/usr/lib/python2.7/dist-packages/neutron/db/common_db_mixin.py", line 130, in _get_by_id\n    return query.filter(model.id == id).one()\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2395, in one\n    ret = list(self)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2437, in __iter__\n    self.session._autoflush()\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1198, in _autoflush\n    self.flush()\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1919, in flush\n    self._flush(objects)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2037, in _flush\n    transaction.rollback(_capture_exception=True)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__\n    compat.reraise(exc_type, exc_value, exc_tb)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2001, in _flush\n    flush_context.execute()\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 372, in execute\n    rec.execute(self)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 526, in execute\n    uow\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 60, in save_obj\n    mapper, table, update)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 518, in _emit_update_statements\n    execute(statement, params)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 729, in execute\n    return meth(self, multiparams, params)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py", line 322, in _execute_on_connection\n    return connection._execute_clauseelement(self, multiparams, params)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 826, in _execute_clauseelement\n    compiled_sql, distilled_params\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 958, in _execute_context\n    context)\n', '  File "/usr/lib/python2.7/dist-packages/oslo_db/sqlalchemy/compat/handle_error.py", line 261, in _handle_dbapi_exception\n    e, statement, parameters, cursor, context)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1155, in _handle_dbapi_exception\n    util.raise_from_cause(newraise, exc_info)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause\n    reraise(type(exception), exception, tb=exc_tb)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 951, in _execute_context\n    context)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 436, in do_execute\n    cursor.execute(statement, parameters)\n', '  File "/usr/lib/python2.7/dist-packages/MySQLdb/cursors.py", line 174, in execute\n    self.errorhandler(self, exc, value)\n', '  File "/usr/lib/python2.7/dist-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler\n    raise errorclass, errorvalue\n', 'DBDuplicateEntry: (IntegrityError) (1062, "Duplicate entry \'234ddb9b-16cc-48f2-b646-fee5f6d5def2\' for key \'PRIMARY\'") \'UPDATE ports SET status=%s WHERE ports.id = %s\' (\'ACTIVE\', \'234ddb9b-16cc-48f2-b646-fee5f6d5def2\')\n']

Some more information about our setup:

* 3 nova/glance/cinder/keystone servers, API endpoints balanced using HAProxy
* 3 MySQL Galera cluster, balanced using HAProxy, but in active/passive configuration (2 over 3 servers as backup in haproxy.cfg)
* 3 RabbitMQ cluster
* All nodes running Ubuntu Trusty and Kilo 2015.1.0 from `ubuntu-cloud.archive.canonical.com`

``neutron.conf`` file:

    [DEFAULT]
    verbose = True
    use_syslog = True
    bind_host = cloud-k4-31.os.s3it.uzh.ch
    core_plugin = ml2
    service_plugins = router
    allow_overlapping_ips = True
    advertise_mtu = True
    dhcp_agents_per_network = 2
    l3_ha = True
    max_l3_agents_per_router = 0
    api_workers = 20
    rpc_workers = 20
    notify_nova_on_port_status_changes = True
    notify_nova_on_port_data_changes = True
    nova_url = http://10.129.17.1:8774/v2
    nova_admin_username = tkunsz
    nova_admin_tenant_name = service
    nova_admin_password = 9.gXd6MpRH
    nova_admin_auth_url = http://130.60.24.21:35357/v2.0
    amqp_durable_queues=false
    rabbit_hosts = cloud-l2-41.os.s3it.uzh.ch,cloud-l6-39.os.s3it.uzh.ch,cloud-k3-39.os.s3it.uzh.ch
    rabbit_ha_queues=True
    notification_driver = neutron.openstack.common.notifier.rpc_notifier
    [matchmaker_redis]
    [matchmaker_ring]
    [quotas]
    [agent]
    root_helper = sudo
    [keystone_authtoken]
    identity_uri = http://130.60.24.21:35357
    admin_tenant_name = service
    admin_user = tkunsz
    admin_password = 9.gXd6MpRH
    auth_uri = http://130.60.24.21:5000/v2.0
    [database]
    connection=mysql://neutron:neutronqwerty@xxxxxxxxxxxxxxxxxxxx/neutron
    max_pool_size = 50
    max_overflow = 100
    [nova]
    region_name = RegionOne
    [oslo_concurrency]
    lock_path = $state_path/lock
    [oslo_policy]
    [oslo_messaging_amqp]
    [oslo_messaging_qpid]
    [oslo_messaging_rabbit]
    amqp_auto_delete = True
    rabbit_hosts = cloud-l2-41.os.s3it.uzh.ch,cloud-l6-39.os.s3it.uzh.ch,cloud-k3-39.os.s3it.uzh.ch
    rabbit_userid=openstack
    rabbit_password=rabbitqwerty
    rabbit_ha_queues = True

``l3_agent.ini`` file:

    [DEFAULT]
    interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
    use_namespaces = True
    external_network_bridge =
    agent_mode = legacy

``dhcp_agent.ini`` file:

    [DEFAULT]
    interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
    dhcp_driver = neutron.agent.linux.dhcp.Dnsmasq
    use_namespaces = True
    enable_isolated_metadata = True

``ml2_conf.ini`` file:

    [ml2]
    type_drivers = gre,vlan,vxlan
    tenant_network_types = vxlan,gre
    mechanism_drivers = openvswitch
    physical_network_mtus = vlannet:9000
    [ml2_type_flat]
    [ml2_type_vlan]
    network_vlan_ranges = vlannet:1:4000
    [ml2_type_gre]
    tunnel_id_ranges = 1:1000
    [ml2_type_vxlan]
    vni_ranges = 65537:69999
    [securitygroup]
    enable_security_group = True
    enable_ipset = True
    firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
    [ovs]
    local_ip = 10.129.31.231
    bridge_mappings = vlannet:br-vlan
    enable_tunneling = True
    tunnel_type = vxlan
    tunnel_types = vxlan,gre
    vxlan_udp_port = 4789
    [agent]
    tunnel_types = vxlan,gre
    l2_population = False

** Affects: neutron
     Importance: Undecided
         Status: New


** Tags: l3-ha network

** Attachment added: "neutron-server-A.log"
   https://bugs.launchpad.net/bugs/1479818/+attachment/4436223/+files/neutron-server-A.log

-- 
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to neutron.
https://bugs.launchpad.net/bugs/1479818

Title:
  "Deadlock found" and "DuplicateEntry" errors when trying to attach a
  neutron L3 HA router to an internal network

Status in neutron:
  New

Bug description:
  On a setup with 2 neutron nodes, L3 agent in HA (VRRP), we get errors
  while attaching a router to an internal network. How to replicate:

  * create the external network (vlan)
  * create an internal network (vxlan)
  * create a router
  * attach the router to the external network
  * attach the router to the internal network

  After attaching the router to the internal network, the following
  happens:

  a) some router _and_ dhcp ports of the external networks go from ACTIVE to BUILD status
  b) dhcp ports of the internal networks go from ACTIVE to BUILD
  c) two type of errors are found on the neutron log file:
     - a **Deadlock found when trying to get a lock** error, on one of the network nodes
     - multiple **Duplicate entry ... for key PRIMARY** on both network nodes

  How we replicated the issue

      root@kenny:~# neutron  net-create uzh-only \
        --shared --provider:network_type vlan \
        --provider:segmentation_id 842 \
        --provider:physical_network vlannet \
        --router:external 

      root@kenny:~# neutron  subnet-create uzh-only \
          --name vlan842 \
          172.23.0.0/16 \
          --gateway 172.23.0.1 \
          --allocation-pool start=172.23.2.0,end=172.23.200.255 \
          --enable-dhcp \
          --dns-nameservers 130.60.128.3 130.60.64.51 \
          --   --host-routes type=dict list=true \
          destination=130.60.206.69/32,nexthop=172.23.0.5

      root@kenny:~# neutron net-create test 
      root@kenny:~# neutron subnet-create test --name subtest 10.0.0.0/24 
      root@kenny:~# neutron router-create test
      root@kenny:~# neutron router-gateway-set test uzh-only

      root@kenny:~# neutron port-list -c id -c fixed_ips -c status
      +--------------------------------------+--------------------------------------------------------------------------------------+--------+-----------------------------+
      | id                                   | fixed_ips                                                                            | status | device_owner                |
      +--------------------------------------+--------------------------------------------------------------------------------------+--------+-----------------------------+
      | 1463b78c-f640-4e41-8831-927679be380b | {"subnet_id": "e5e43019-58e4-4203-a51b-8e0dce7ca829", "ip_address": "10.0.0.3"}      | ACTIVE | network:dhcp                |
      | 1e7a0ac2-268c-48b7-9c70-e7e03126d9ed | {"subnet_id": "178f49ec-383e-4516-b6c3-814e7f459497", "ip_address": "169.254.192.1"} | ACTIVE | network:router_ha_interface |
      | 234ddb9b-16cc-48f2-b646-fee5f6d5def2 | {"subnet_id": "88a38e12-9402-418f-b0b7-1deeb843edf2", "ip_address": "172.23.2.2"}    | BUILD  | network:router_gateway      |
      | 3e755174-14b3-48e9-af84-489cce8985be | {"subnet_id": "e5e43019-58e4-4203-a51b-8e0dce7ca829", "ip_address": "10.0.0.2"}      | ACTIVE | network:dhcp                |
      | 4df3d3ab-1967-4bf1-bc3f-31ec1addd8d1 | {"subnet_id": "e5e43019-58e4-4203-a51b-8e0dce7ca829", "ip_address": "10.0.0.1"}      | BUILD  | network:router_interface    |
      | 5a2e6cbd-4c69-47e0-bfa6-92fe6242aba4 | {"subnet_id": "88a38e12-9402-418f-b0b7-1deeb843edf2", "ip_address": "172.23.2.0"}    | ACTIVE | network:dhcp                |
      | 881882f0-4ad7-4294-bec4-93f869cb40d8 | {"subnet_id": "178f49ec-383e-4516-b6c3-814e7f459497", "ip_address": "169.254.192.2"} | ACTIVE | network:router_ha_interface |
      | ac0e1761-335e-431f-ae89-a2703f2bafa5 | {"subnet_id": "88a38e12-9402-418f-b0b7-1deeb843edf2", "ip_address": "172.23.2.1"}    | ACTIVE | network:dhcp                |
      +--------------------------------------+--------------------------------------------------------------------------------------+--------+-----------------------------+

      root@kenny:~# neutron router-interface-add test subtest

      root@kenny:~# neutron port-list -c id -c fixed_ips -c status

      root@kenny:~# neutron port-list -c id -c fixed_ips -c status -c device_owner
      +--------------------------------------+--------------------------------------------------------------------------------------+--------+-----------------------------+
      | id                                   | fixed_ips                                                                            | status | device_owner                |
      +--------------------------------------+--------------------------------------------------------------------------------------+--------+-----------------------------+
      | 1463b78c-f640-4e41-8831-927679be380b | {"subnet_id": "e5e43019-58e4-4203-a51b-8e0dce7ca829", "ip_address": "10.0.0.3"}      | BUILD  | network:dhcp                |
      | 1e7a0ac2-268c-48b7-9c70-e7e03126d9ed | {"subnet_id": "178f49ec-383e-4516-b6c3-814e7f459497", "ip_address": "169.254.192.1"} | ACTIVE | network:router_ha_interface |
      | 234ddb9b-16cc-48f2-b646-fee5f6d5def2 | {"subnet_id": "88a38e12-9402-418f-b0b7-1deeb843edf2", "ip_address": "172.23.2.2"}    | BUILD  | network:router_gateway      |
      | 3e755174-14b3-48e9-af84-489cce8985be | {"subnet_id": "e5e43019-58e4-4203-a51b-8e0dce7ca829", "ip_address": "10.0.0.2"}      | ACTIVE | network:dhcp                |
      | 4df3d3ab-1967-4bf1-bc3f-31ec1addd8d1 | {"subnet_id": "e5e43019-58e4-4203-a51b-8e0dce7ca829", "ip_address": "10.0.0.1"}      | BUILD  | network:router_interface    |
      | 5a2e6cbd-4c69-47e0-bfa6-92fe6242aba4 | {"subnet_id": "88a38e12-9402-418f-b0b7-1deeb843edf2", "ip_address": "172.23.2.0"}    | ACTIVE | network:dhcp                |
      | 881882f0-4ad7-4294-bec4-93f869cb40d8 | {"subnet_id": "178f49ec-383e-4516-b6c3-814e7f459497", "ip_address": "169.254.192.2"} | BUILD  | network:router_ha_interface |
      | ac0e1761-335e-431f-ae89-a2703f2bafa5 | {"subnet_id": "88a38e12-9402-418f-b0b7-1deeb843edf2", "ip_address": "172.23.2.1"}    | BUILD  | network:dhcp                |
      +--------------------------------------+--------------------------------------------------------------------------------------+--------+-----------------------------+

  excerpt from the logs on controller node A (only WARNING, ERROR and
  TRACE)

      2015-07-30 15:50:59.322 251400 WARNING neutron.api.rpc.handlers.dhcp_rpc [req-203d6d37-4643-46e1-baa4-eb58c5f45750 ] Action create_port for network 9f7e9af9-72fc-44f1-89e3-be5cdf3d5bc5 could not complete successfully: (OperationalError) (1213, 'Deadlock found when trying to get lock; try restarting transaction') 'INSERT INTO ml2_port_binding_levels (port_id, host, level, driver, segment_id) VALUES (%s, %s, %s, %s, %s)' ('3e755174-14b3-48e9-af84-489cce8985be', 'cloud-l7-41', 0, 'openvswitch', '9a00e2d3-d842-4517-bec2-e3296ae711a5')
      2015-07-30 15:51:46.687 228137 ERROR neutron.plugins.openvswitch.agent.ovs_neutron_agent [req-0c92ed7e-f3c8-4d56-92dd-6ee597d25143 ] Error while processing VIF ports
      2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent Traceback (most recent call last):
      2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py", line 1586, in rpc_loop
      2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     ovs_restarted)
      2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py", line 1362, in process_network_ports
      2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     devices_added_updated, ovs_restarted)
      2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py", line 1267, in treat_devices_added_or_updated
      2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     self.context, device, self.agent_id, cfg.CONF.host)
      2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/neutron/agent/rpc.py", line 128, in update_device_up
      2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     agent_id=agent_id, host=host)
      2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/client.py", line 156, in call
      2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     retry=self.retry)
      2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/oslo_messaging/transport.py", line 90, in _send
      2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     timeout=timeout, retry=retry)
      2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py", line 350, in send
      2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     retry=retry)
      2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent   File "/usr/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py", line 341, in _send
      2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent     raise result
      2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent RemoteError: Remote error: DBDuplicateEntry (IntegrityError) (1062, "Duplicate entry '234ddb9b-16cc-48f2-b646-fee5f6d5def2' for key 'PRIMARY'") 'UPDATE ports SET status=%s WHERE ports.id = %s' ('ACTIVE', '234ddb9b-16cc-48f2-b646-fee5f6d5def2')
      2015-07-30 15:51:46.687 228137 TRACE neutron.plugins.openvswitch.agent.ovs_neutron_agent [u'Traceback (most recent call last):\n', u'  File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 142, in _dispatch_and_reply\n    executor_callback))\n', u'  File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 186, in _dispatch\n    executor_callback)\n', u'  File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 130, in _do_dispatch\n    result = func(ctxt, **new_args)\n', u'  File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/rpc.py", line 186, in update_device_up\n    host)\n', u'  File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 1373, in update_port_status\n    original_port[\'network_id\'])\n', u'  File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 658, in get_network\n    result = super(Ml2Plugin, self).get_network(context, id, None)\n', u'  File "/usr/lib/python2.7/dist-packages/neutron/db/db_base_plugin_v2.py", line 1007, in get_network\n    network = self._get_network(context, id)\n', u'  File "/usr/lib/python2.7/dist-packages/neutron/db/db_base_plugin_v2.py", line 91, in _get_network\n    network = self._get_by_id(context, models_v2.Network, id)\n', u'  File "/usr/lib/python2.7/dist-packages/neutron/db/common_db_mixin.py", line 130, in _get_by_id\n    return query.filter(model.id == id).one()\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2395, in one\n    ret = list(self)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2437, in __iter__\n    self.session._autoflush()\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1198, in _autoflush\n    self.flush()\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1919, in flush\n    self._flush(objects)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2037, in _flush\n    transaction.rollback(_capture_exception=True)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__\n    compat.reraise(exc_type, exc_value, exc_tb)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2001, in _flush\n    flush_context.execute()\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 372, in execute\n    rec.execute(self)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 526, in execute\n    uow\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 60, in save_obj\n    mapper, table, update)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 518, in _emit_update_statements\n    execute(statement, params)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 729, in execute\n    return meth(self, multiparams, params)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py", line 322, in _execute_on_connection\n    return connection._execute_clauseelement(self, multiparams, params)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 826, in _execute_clauseelement\n    compiled_sql, distilled_params\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 958, in _execute_context\n    context)\n', u'  File "/usr/lib/python2.7/dist-packages/oslo_db/sqlalchemy/compat/handle_error.py", line 261, in _handle_dbapi_exception\n    e, statement, parameters, cursor, context)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1155, in _handle_dbapi_exception\n    util.raise_from_cause(newraise, exc_info)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause\n    reraise(type(exception), exception, tb=exc_tb)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 951, in _execute_context\n    context)\n', u'  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 436, in do_execute\n    cursor.execute(statement, parameters)\n', u'  File "/usr/lib/python2.7/dist-packages/MySQLdb/cursors.py", line 174, in execute\n    self.errorhandler(self, exc, value)\n', u'  File "/usr/lib/python2.7/dist-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler\n    raise errorclass, errorvalue\n', u'DBDuplicateEntry: (IntegrityError) (1062, "Duplicate entry \'234ddb9b-16cc-48f2-b646-fee5f6d5def2\' for key \'PRIMARY\'") \'UPDATE ports SET status=%s WHERE ports.id = %s\' (\'ACTIVE\', \'234ddb9b-16cc-48f2-b646-fee5f6d5def2\')\n'].

  excerpt from the logs on controller node B (only WARNING, ERROR and
  TRACE)

      2015-07-30 15:51:46.680 55894 ERROR oslo_messaging.rpc.dispatcher [req-0c92ed7e-f3c8-4d56-92dd-6ee597d25143 ] Exception during message handling: (IntegrityError) (1062, "Duplicate entry '234ddb9b-16cc-48f2-b646-fee5f6d5def2' for key 'PRIMARY'") 'UPDATE ports SET status=%s WHERE ports.id = %s' ('ACTIVE', '234ddb9b-16cc-48f2-b646-fee5f6d5def2')
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher Traceback (most recent call last):
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 142, in _dispatch_and_reply
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     executor_callback))
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 186, in _dispatch
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     executor_callback)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 130, in _do_dispatch
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     result = func(ctxt, **new_args)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/rpc.py", line 186, in update_device_up
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     host)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 1373, in update_port_status
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     original_port['network_id'])
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 658, in get_network
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     result = super(Ml2Plugin, self).get_network(context, id, None)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/neutron/db/db_base_plugin_v2.py", line 1007, in get_network
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     network = self._get_network(context, id)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/neutron/db/db_base_plugin_v2.py", line 91, in _get_network
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     network = self._get_by_id(context, models_v2.Network, id)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/neutron/db/common_db_mixin.py", line 130, in _get_by_id
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     return query.filter(model.id == id).one()
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2395, in one
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     ret = list(self)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2437, in __iter__
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     self.session._autoflush()
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1198, in _autoflush
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     self.flush()
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1919, in flush
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     self._flush(objects)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2037, in _flush
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     transaction.rollback(_capture_exception=True)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     compat.reraise(exc_type, exc_value, exc_tb)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2001, in _flush
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     flush_context.execute()
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 372, in execute
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     rec.execute(self)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 526, in execute
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     uow
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 60, in save_obj
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     mapper, table, update)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 518, in _emit_update_statements
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     execute(statement, params)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 729, in execute
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     return meth(self, multiparams, params)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py", line 322, in _execute_on_connection
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     return connection._execute_clauseelement(self, multiparams, params)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 826, in _execute_clauseelement
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     compiled_sql, distilled_params
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 958, in _execute_context
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     context)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/oslo_db/sqlalchemy/compat/handle_error.py", line 261, in _handle_dbapi_exception
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     e, statement, parameters, cursor, context)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1155, in _handle_dbapi_exception
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     util.raise_from_cause(newraise, exc_info)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     reraise(type(exception), exception, tb=exc_tb)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 951, in _execute_context
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     context)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 436, in do_execute
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     cursor.execute(statement, parameters)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/MySQLdb/cursors.py", line 174, in execute
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     self.errorhandler(self, exc, value)
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher   File "/usr/lib/python2.7/dist-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher     raise errorclass, errorvalue
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher DBDuplicateEntry: (IntegrityError) (1062, "Duplicate entry '234ddb9b-16cc-48f2-b646-fee5f6d5def2' for key 'PRIMARY'") 'UPDATE ports SET status=%s WHERE ports.id = %s' ('ACTIVE', '234ddb9b-16cc-48f2-b646-fee5f6d5def2')
      2015-07-30 15:51:46.680 55894 TRACE oslo_messaging.rpc.dispatcher 
      2015-07-30 15:51:46.682 55894 ERROR oslo_messaging._drivers.common [req-0c92ed7e-f3c8-4d56-92dd-6ee597d25143 ] Returning exception (IntegrityError) (1062, "Duplicate entry '234ddb9b-16cc-48f2-b646-fee5f6d5def2' for key 'PRIMARY'") 'UPDATE ports SET status=%s WHERE ports.id = %s' ('ACTIVE', '234ddb9b-16cc-48f2-b646-fee5f6d5def2') to caller
      2015-07-30 15:51:46.682 55894 ERROR oslo_messaging._drivers.common [req-0c92ed7e-f3c8-4d56-92dd-6ee597d25143 ] ['Traceback (most recent call last):\n', '  File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 142, in _dispatch_and_reply\n    executor_callback))\n', '  File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 186, in _dispatch\n    executor_callback)\n', '  File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 130, in _do_dispatch\n    result = func(ctxt, **new_args)\n', '  File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/rpc.py", line 186, in update_device_up\n    host)\n', '  File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 1373, in update_port_status\n    original_port[\'network_id\'])\n', '  File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line 658, in get_network\n    result = super(Ml2Plugin, self).get_network(context, id, None)\n', '  File "/usr/lib/python2.7/dist-packages/neutron/db/db_base_plugin_v2.py", line 1007, in get_network\n    network = self._get_network(context, id)\n', '  File "/usr/lib/python2.7/dist-packages/neutron/db/db_base_plugin_v2.py", line 91, in _get_network\n    network = self._get_by_id(context, models_v2.Network, id)\n', '  File "/usr/lib/python2.7/dist-packages/neutron/db/common_db_mixin.py", line 130, in _get_by_id\n    return query.filter(model.id == id).one()\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2395, in one\n    ret = list(self)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2437, in __iter__\n    self.session._autoflush()\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1198, in _autoflush\n    self.flush()\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1919, in flush\n    self._flush(objects)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2037, in _flush\n    transaction.rollback(_capture_exception=True)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__\n    compat.reraise(exc_type, exc_value, exc_tb)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2001, in _flush\n    flush_context.execute()\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 372, in execute\n    rec.execute(self)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 526, in execute\n    uow\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 60, in save_obj\n    mapper, table, update)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 518, in _emit_update_statements\n    execute(statement, params)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 729, in execute\n    return meth(self, multiparams, params)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py", line 322, in _execute_on_connection\n    return connection._execute_clauseelement(self, multiparams, params)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 826, in _execute_clauseelement\n    compiled_sql, distilled_params\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 958, in _execute_context\n    context)\n', '  File "/usr/lib/python2.7/dist-packages/oslo_db/sqlalchemy/compat/handle_error.py", line 261, in _handle_dbapi_exception\n    e, statement, parameters, cursor, context)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1155, in _handle_dbapi_exception\n    util.raise_from_cause(newraise, exc_info)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause\n    reraise(type(exception), exception, tb=exc_tb)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 951, in _execute_context\n    context)\n', '  File "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 436, in do_execute\n    cursor.execute(statement, parameters)\n', '  File "/usr/lib/python2.7/dist-packages/MySQLdb/cursors.py", line 174, in execute\n    self.errorhandler(self, exc, value)\n', '  File "/usr/lib/python2.7/dist-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler\n    raise errorclass, errorvalue\n', 'DBDuplicateEntry: (IntegrityError) (1062, "Duplicate entry \'234ddb9b-16cc-48f2-b646-fee5f6d5def2\' for key \'PRIMARY\'") \'UPDATE ports SET status=%s WHERE ports.id = %s\' (\'ACTIVE\', \'234ddb9b-16cc-48f2-b646-fee5f6d5def2\')\n']

  Some more information about our setup:

  * 3 nova/glance/cinder/keystone servers, API endpoints balanced using HAProxy
  * 3 MySQL Galera cluster, balanced using HAProxy, but in active/passive configuration (2 over 3 servers as backup in haproxy.cfg)
  * 3 RabbitMQ cluster
  * All nodes running Ubuntu Trusty and Kilo 2015.1.0 from `ubuntu-cloud.archive.canonical.com`

  ``neutron.conf`` file:

      [DEFAULT]
      verbose = True
      use_syslog = True
      bind_host = cloud-k4-31.os.s3it.uzh.ch
      core_plugin = ml2
      service_plugins = router
      allow_overlapping_ips = True
      advertise_mtu = True
      dhcp_agents_per_network = 2
      l3_ha = True
      max_l3_agents_per_router = 0
      api_workers = 20
      rpc_workers = 20
      notify_nova_on_port_status_changes = True
      notify_nova_on_port_data_changes = True
      nova_url = http://10.129.17.1:8774/v2
      nova_admin_username = tkunsz
      nova_admin_tenant_name = service
      nova_admin_password = 9.gXd6MpRH
      nova_admin_auth_url = http://130.60.24.21:35357/v2.0
      amqp_durable_queues=false
      rabbit_hosts = cloud-l2-41.os.s3it.uzh.ch,cloud-l6-39.os.s3it.uzh.ch,cloud-k3-39.os.s3it.uzh.ch
      rabbit_ha_queues=True
      notification_driver = neutron.openstack.common.notifier.rpc_notifier
      [matchmaker_redis]
      [matchmaker_ring]
      [quotas]
      [agent]
      root_helper = sudo
      [keystone_authtoken]
      identity_uri = http://130.60.24.21:35357
      admin_tenant_name = service
      admin_user = tkunsz
      admin_password = 9.gXd6MpRH
      auth_uri = http://130.60.24.21:5000/v2.0
      [database]
      connection=mysql://neutron:neutronqwerty@xxxxxxxxxxxxxxxxxxxx/neutron
      max_pool_size = 50
      max_overflow = 100
      [nova]
      region_name = RegionOne
      [oslo_concurrency]
      lock_path = $state_path/lock
      [oslo_policy]
      [oslo_messaging_amqp]
      [oslo_messaging_qpid]
      [oslo_messaging_rabbit]
      amqp_auto_delete = True
      rabbit_hosts = cloud-l2-41.os.s3it.uzh.ch,cloud-l6-39.os.s3it.uzh.ch,cloud-k3-39.os.s3it.uzh.ch
      rabbit_userid=openstack
      rabbit_password=rabbitqwerty
      rabbit_ha_queues = True

  ``l3_agent.ini`` file:

      [DEFAULT]
      interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
      use_namespaces = True
      external_network_bridge =
      agent_mode = legacy

  ``dhcp_agent.ini`` file:

      [DEFAULT]
      interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
      dhcp_driver = neutron.agent.linux.dhcp.Dnsmasq
      use_namespaces = True
      enable_isolated_metadata = True

  ``ml2_conf.ini`` file:

      [ml2]
      type_drivers = gre,vlan,vxlan
      tenant_network_types = vxlan,gre
      mechanism_drivers = openvswitch
      physical_network_mtus = vlannet:9000
      [ml2_type_flat]
      [ml2_type_vlan]
      network_vlan_ranges = vlannet:1:4000
      [ml2_type_gre]
      tunnel_id_ranges = 1:1000
      [ml2_type_vxlan]
      vni_ranges = 65537:69999
      [securitygroup]
      enable_security_group = True
      enable_ipset = True
      firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
      [ovs]
      local_ip = 10.129.31.231
      bridge_mappings = vlannet:br-vlan
      enable_tunneling = True
      tunnel_type = vxlan
      tunnel_types = vxlan,gre
      vxlan_udp_port = 4789
      [agent]
      tunnel_types = vxlan,gre
      l2_population = False

To manage notifications about this bug go to:
https://bugs.launchpad.net/neutron/+bug/1479818/+subscriptions


Follow ups