openstack team mailing list archive

Thread
Date
OpenStack Multi Node Problems with OVS

To: "openstack@xxxxxxxxxxxxxxxxxxx" <openstack@xxxxxxxxxxxxxxxxxxx>
From: Atif Wasi <awasi@xxxxxxxxx>
Date: Tue, 16 Apr 2013 19:14:15 -0700 (PDT)
In-reply-to: <CAHcn5b3cQEvaZ6vpQM_Yi+Dd_TkAngqYoxBfWx-WLt65PCq3NA@mail.gmail.com>
Reply-to: Atif Wasi <awasi@xxxxxxxxx>
Hi Folks,

I am working on bringing up a multi node OpenStack environment with OVS.  I have a Controller, Compute and a Gateway/Network node.  This is running Folsom.  Most of the services are up, except that I cannot ping the floating ip of the VM.  
Below are the error messages I am seeing the Gateway/Network node:

root@gateway:/var/log/quantum# ovs-vsctl show
9f3f021a-cb8e-4ef4-87a5-f77775bab9f3
    Bridge br-ex
        Port "eth2"
            Interface "eth2"
        Port "qg-0aff855e-9c"
            Interface "qg-0aff855e-9c"
                type: internal
        Port br-ex
            Interface br-ex
                type: internal
    Bridge "br-eth1"
        Port "eth1"
            Interface "eth1"
        Port "phy-br-eth1"
            Interface "phy-br-eth1"
        Port "br-eth1"
            Interface "br-eth1"
                type: internal
    Bridge br-int
        Port br-int
            Interface br-int
                type: internal
        Port "int-br-eth1"
            Interface "int-br-eth1"
        Port "tap47ac110a-fe"
            tag: 1
            Interface "tap47ac110a-fe"
                type: internal
        Port "qr-12091c58-8a"
            tag: 1
            Interface "qr-12091c58-8a"
                type: internal
    ovs_version: "1.4.3"

=====================================
L3 Agent.log:

2013-04-16 22:06:08    DEBUG [quantum.agent.linux.utils] Running command: ip -o link show br-ex
2013-04-16 22:06:08    DEBUG [quantum.agent.linux.utils] 
Command: ['ip', '-o', 'link', 'show', 'br-ex']
Exit code: 0
Stdout: '20: br-ex: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN mode DEFAULT \\    link/ether 00:04:23:c9:64:ac brd ff:ff:ff:ff:ff:ff\n'
Stderr: ''
2013-04-16 22:06:08    DEBUG [quantumclient.client] REQ: curl -i http://192.168.0.2:9696//v2.0/networks.json?router%3Aexternal=True -X GET -H "User-Agent: python-quantumclient" -H "Content-Type: application/json" -H "Accept: application/json" -H "X-Auth-Token: 3c2cc4afb68f4042bf4bddd9908186d0"

2013-04-16 22:06:08    DEBUG [quantumclient.client] RESP:{'date': 'Wed, 17 Apr 2013 02:06:08 GMT', 'status': '200', 'content-length': '376', 'content-type': 'application/json', 'content-location': u'http://192.168.0.2:9696//v2.0/networks.json?router%3Aexternal=True'}

2013-04-16 22:06:08    DEBUG [quantumclient.client] RESP BODY:{"networks": [{"status": "ACTIVE", "subnets": ["487ccc01-81ea-4abc-bbb4-061cde722801"], "name": "ext_net", "provider:physical_network": "physnet1", "admin_state_up": true, "tenant_id": "6bd416e48a394f06bfbf4d0856740262", "provider:network_type": "vlan", "router:external": true, "shared": false, "id": "2f2aa2c9-47be-4b3e-9f05-a547a8d3aaa9", "provider:segmentation_id": 500}]}

2013-04-16 22:06:08    DEBUG [quantumclient.client] REQ: curl -i http://192.168.0.2:9696//v2.0/routers.json -X GET -H "User-Agent: python-quantumclient" -H "Content-Type: application/json" -H "Accept: application/json" -H "X-Auth-Token: 3c2cc4afb68f4042bf4bddd9908186d0"

2013-04-16 22:06:08    DEBUG [quantumclient.client] RESP:{'date': 'Wed, 17 Apr 2013 02:06:08 GMT', 'status': '200', 'content-length': '262', 'content-type': 'application/json', 'content-location': u'http://192.168.0.2:9696//v2.0/routers.json'}

2013-04-16 22:06:08    DEBUG [quantumclient.client] RESP BODY:{"routers": [{"status": "ACTIVE", "external_gateway_info": {"network_id": "2f2aa2c9-47be-4b3e-9f05-a547a8d3aaa9"}, "name": "router_proj_one", "admin_state_up": true, "tenant_id": "b443045cd0d54377b637cc0ba998bf89", "id": "49a92bbc-5f58-400a-a8cb-aebcf54db446"}]}

2013-04-16 22:06:08    DEBUG [quantumclient.client] REQ: curl -i http://192.168.0.2:9696//v2.0/ports.json?device_owner=network%3Arouter_gateway&device_id=49a92bbc-5f58-400a-a8cb-aebcf54db446 -X GET -H "User-Agent: python-quantumclient" -H "Content-Type: application/json" -H "Accept: application/json" -H "X-Auth-Token: 3c2cc4afb68f4042bf4bddd9908186d0"

2013-04-16 22:06:08    DEBUG [quantumclient.client] RESP:{'date': 'Wed, 17 Apr 2013 02:06:08 GMT', 'status': '200', 'content-length': '417', 'content-type': 'application/json', 'content-location': u'http://192.168.0.2:9696//v2.0/ports.json?device_owner=network%3Arouter_gateway&device_id=49a92bbc-5f58-400a-a8cb-aebcf54db446'}

2013-04-16 22:06:08    DEBUG [quantumclient.client] RESP BODY:{"ports": [{"status": "ACTIVE", "name": "", "admin_state_up": true, "network_id": "2f2aa2c9-47be-4b3e-9f05-a547a8d3aaa9", "tenant_id": "", "device_owner": "network:router_gateway", "mac_address": "fa:16:3e:3a:fe:6a", "fixed_ips": [{"subnet_id": "487ccc01-81ea-4abc-bbb4-061cde722801", "ip_address": "192.168.1.10"}], "id": "0aff855e-9cab-4dad-a490-7019e104603a", "device_id": "49a92bbc-5f58-400a-a8cb-aebcf54db446"}]}

2013-04-16 22:06:08    DEBUG [quantumclient.client] REQ: curl -i http://192.168.0.2:9696//v2.0/ports.json?device_owner=network%3Arouter_interface&device_id=49a92bbc-5f58-400a-a8cb-aebcf54db446 -X GET -H "User-Agent: python-quantumclient" -H "Content-Type: application/json" -H "Accept: application/json" -H "X-Auth-Token: 3c2cc4afb68f4042bf4bddd9908186d0"

2013-04-16 22:06:08    DEBUG [quantumclient.client] RESP:{'date': 'Wed, 17 Apr 2013 02:06:09 GMT', 'status': '200', 'content-length': '452', 'content-type': 'application/json', 'content-location': u'http://192.168.0.2:9696//v2.0/ports.json?device_owner=network%3Arouter_interface&device_id=49a92bbc-5f58-400a-a8cb-aebcf54db446'}

2013-04-16 22:06:08    DEBUG [quantumclient.client] RESP BODY:{"ports": [{"status": "ACTIVE", "name": "", "admin_state_up": true, "network_id": "53e1bcd2-d107-4236-aa3a-718bfa641ead", "tenant_id": "b443045cd0d54377b637cc0ba998bf89", "device_owner": "network:router_interface", "mac_address": "fa:16:3e:52:8b:d5", "fixed_ips": [{"subnet_id": "47e9644b-c536-4662-8771-2f296d47ad4d", "ip_address": "192.168.100.1"}], "id": "12091c58-8a04-4b0d-bf8a-53575e16298a", "device_id": "49a92bbc-5f58-400a-a8cb-aebcf54db446"}]}

2013-04-16 22:06:08    DEBUG [quantumclient.client] REQ: curl -i http://192.168.0.2:9696//v2.0/floatingips.json?router_id=49a92bbc-5f58-400a-a8cb-aebcf54db446 -X GET -H "User-Agent: python-quantumclient" -H "Content-Type: application/json" -H "Accept: application/json" -H "X-Auth-Token: 3c2cc4afb68f4042bf4bddd9908186d0"

2013-04-16 22:06:08    DEBUG [quantumclient.client] RESP:{'date': 'Wed, 17 Apr 2013 02:06:09 GMT', 'status': '200', 'content-length': '357', 'content-type': 'application/json', 'content-location': u'http://192.168.0.2:9696//v2.0/floatingips.json?router_id=49a92bbc-5f58-400a-a8cb-aebcf54db446'}

2013-04-16 22:06:08    DEBUG [quantumclient.client] RESP BODY:{"floatingips": [{"router_id": "49a92bbc-5f58-400a-a8cb-aebcf54db446", "tenant_id": "b443045cd0d54377b637cc0ba998bf89", "floating_network_id": "2f2aa2c9-47be-4b3e-9f05-a547a8d3aaa9", "fixed_ip_address": "192.168.100.3", "floating_ip_address": "192.168.1.11", "port_id": "015cfbe5-9310-43cd-8edf-321d5907fa56", "id": "0613c05f-8ae4-4596-a0fa-35e72c96c6e4"}]}

==============================================

openvswitch-agent.log

2013-04-16 22:06:50    DEBUG [quantum.agent.linux.utils] Running command: sudo /usr/bin/quantum-rootwrap /etc/quantum/rootwrap.conf ovs-vsctl --timeout=2 list-ports br-int
2013-04-16 22:06:50    DEBUG [quantum.agent.linux.utils] 
Command: ['sudo', '/usr/bin/quantum-rootwrap', '/etc/quantum/rootwrap.conf', 'ovs-vsctl', '--timeout=2', 'list-ports', 'br-int']
Exit code: 0
Stdout: 'int-br-eth1\nqr-12091c58-8a\ntap47ac110a-fe\n'
Stderr: ''
2013-04-16 22:06:50    DEBUG [quantum.agent.linux.utils] Running command: sudo /usr/bin/quantum-rootwrap /etc/quantum/rootwrap.conf ovs-vsctl --timeout=2 get Interface int-br-eth1 external_ids
2013-04-16 22:06:50    DEBUG [quantum.agent.linux.utils] 
Command: ['sudo', '/usr/bin/quantum-rootwrap', '/etc/quantum/rootwrap.conf', 'ovs-vsctl', '--timeout=2', 'get', 'Interface', 'int-br-eth1', 'external_ids']
Exit code: 0
Stdout: '{}\n'
Stderr: ''
2013-04-16 22:06:50    DEBUG [quantum.agent.linux.utils] Running command: sudo /usr/bin/quantum-rootwrap /etc/quantum/rootwrap.conf ovs-vsctl --timeout=2 get Interface qr-12091c58-8a external_ids
2013-04-16 22:06:50    DEBUG [quantum.agent.linux.utils] 
Command: ['sudo', '/usr/bin/quantum-rootwrap', '/etc/quantum/rootwrap.conf', 'ovs-vsctl', '--timeout=2', 'get', 'Interface', 'qr-12091c58-8a', 'external_ids']
Exit code: 0
Stdout: '{attached-mac="fa:16:3e:52:8b:d5", iface-id="12091c58-8a04-4b0d-bf8a-53575e16298a", iface-status=active}\n'
Stderr: ''
2013-04-16 22:06:50    DEBUG [quantum.agent.linux.utils] Running command: sudo /usr/bin/quantum-rootwrap /etc/quantum/rootwrap.conf ovs-vsctl --timeout=2 get Interface tap47ac110a-fe external_ids
2013-04-16 22:06:51    DEBUG [quantum.agent.linux.utils] 
Command: ['sudo', '/usr/bin/quantum-rootwrap', '/etc/quantum/rootwrap.conf', 'ovs-vsctl', '--timeout=2', 'get', 'Interface', 'tap47ac110a-fe', 'external_ids']
Exit code: 0
Stdout: '{attached-mac="fa:16:3e:02:f5:f9", iface-id="47ac110a-fe25-43ec-b83a-4db58071a3d3", iface-status=active}\n'
Stderr: ''

=============================================================

root@gateway:/var/log/quantum# sudo /usr/bin/quantum-rootwrap /etc/quantum/rootwrap.conf ovs-vsctl --timeout=2 list-ports br-int
int-br-eth1
qr-12091c58-8a
tap47ac110a-fe


============================================================================


ovs-vswitchd.log

Apr 16 22:08:39|00482|netdev|WARN|Dropped 233 log messages in last 12 seconds (most recently, 1 seconds ago) due to excessive rate
Apr 16 22:08:39|00483|netdev|WARN|failed to get flags for network device qr-12091c58-8a: No such device
Apr 16 22:08:43|00484|netdev_linux|WARN|Dropped 35 log messages in last 15 seconds (most recently, 5 seconds ago) due to excessive rate
Apr 16 22:08:43|00485|netdev_linux|WARN|ioctl(SIOCGIFINDEX) on qr-12091c58-8a device failed: No such device
Apr 16 22:08:52|00486|netdev|WARN|Dropped 233 log messages in last 12 seconds (most recently, 1 seconds ago) due to excessive rate
Apr 16 22:08:52|00487|netdev|WARN|failed to get flags for network device qr-12091c58-8a: No such device
Apr 16 22:08:53|00488|netdev_linux|WARN|Dropped 23 log messages in last 10 seconds (most recently, 5 seconds ago) due to excessive rate
Apr 16 22:08:53|00489|netdev_linux|WARN|ioctl(SIOCGIFINDEX) on qr-12091c58-8a device failed: No such device
Apr 16 22:09:04|00490|netdev|WARN|Dropped 227 log messages in last 12 seconds (most recently, 1 seconds ago) due to excessive rate
Apr 16 22:09:04|00491|netdev|WARN|failed to get flags for network device qr-12091c58-8a: No such device

=====================================

oot@gateway:/var/log/quantum# ifconfig -a
br-eth1   Link encap:Ethernet  HWaddr 00:13:72:55:86:33  
          BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:728 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0 
          RX bytes:172923 (172.9 KB)  TX bytes:0 (0.0 B)

br-ex     Link encap:Ethernet  HWaddr 00:04:23:c9:64:ac  
          BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:695 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0 
          RX bytes:161089 (161.0 KB)  TX bytes:0 (0.0 B)

br-int    Link encap:Ethernet  HWaddr 2a:e5:80:ce:aa:43  
          BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:23 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0 
          RX bytes:4496 (4.4 KB)  TX bytes:0 (0.0 B)

eth0      Link encap:Ethernet  HWaddr 00:13:72:55:86:32  
          inet addr:192.168.0.3  Bcast:192.168.0.255  Mask:255.255.255.0
          inet6 addr: fe80::213:72ff:fe55:8632/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:5262 errors:0 dropped:0 overruns:0 frame:0
          TX packets:3830 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:975352 (975.3 KB)  TX bytes:816388 (816.3 KB)

eth1      Link encap:Ethernet  HWaddr 00:13:72:55:86:33  
          inet addr:10.10.10.3  Bcast:10.10.10.255  Mask:255.255.255.0
          inet6 addr: fe80::213:72ff:fe55:8633/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:1690 errors:0 dropped:0 overruns:0 frame:0
          TX packets:56 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:327902 (327.9 KB)  TX bytes:8684 (8.6 KB)

eth2      Link encap:Ethernet  HWaddr 00:04:23:c9:64:ac  
          inet addr:192.168.1.5  Bcast:192.168.1.255  Mask:255.255.255.0
          inet6 addr: fe80::204:23ff:fec9:64ac/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:1664 errors:0 dropped:0 overruns:0 frame:0
          TX packets:423 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:318659 (318.6 KB)  TX bytes:26402 (26.4 KB)

eth3      Link encap:Ethernet  HWaddr 00:04:23:c9:64:ad  
          BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

int-br-eth1 Link encap:Ethernet  HWaddr 7e:42:53:c7:c4:f2  
          inet6 addr: fe80::7c42:53ff:fec7:c4f2/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:698 errors:0 dropped:0 overruns:0 frame:0
          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:164330 (164.3 KB)  TX bytes:5955 (5.9 KB)

lo        Link encap:Local Loopback  
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:16436  Metric:1
          RX packets:194 errors:0 dropped:0 overruns:0 frame:0
          TX packets:194 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0 
          RX bytes:18778 (18.7 KB)  TX bytes:18778 (18.7 KB)

phy-br-eth1 Link encap:Ethernet  HWaddr 3a:a7:27:ec:28:4e  
          inet6 addr: fe80::38a7:27ff:feec:284e/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:39 errors:0 dropped:0 overruns:0 frame:0
          TX packets:698 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:5955 (5.9 KB)  TX bytes:164330 (164.3 KB)
Follow ups

Re: OpenStack Multi Node Problems with OVS
From: Robert van Leeuwen, 2013-04-17
Re: OpenStack Multi Node Problems with OVS
From: Steve Heistand, 2013-04-17
References

[Doc] Talk shop: API Docs
From: Anne Gentle, 2013-04-16
Re: [Doc] Talk shop: API Docs
From: thingee, 2013-04-16