openstack team mailing list archive

Thread
Date
Re: OpenStack Multi Node Problems with OVS

To: Atif Wasi <awasi@xxxxxxxxx>, Atif Wasi <awasi@xxxxxxxxx>, "openstack@xxxxxxxxxxxxxxxxxxx" <openstack@xxxxxxxxxxxxxxxxxxx>
From: Steve Heistand <steve.heistand@xxxxxxxx>
Date: Tue, 16 Apr 2013 19:19:48 -0700
In-reply-to: <1366164855.73505.YahooMailNeo@web84505.mail.ne1.yahoo.com>
User-agent: K-9 Mail for Android
Welcome to the club :) 
Been fighting the same problem for a week now. Nothing is working...
Found lots of other people complaining on the web even with grizzly like I'm using.

S


Atif Wasi <awasi@xxxxxxxxx> wrote:

>Hi Folks,
>
>
>I am working on bringing up a multi node OpenStack environment with
>OVS.  I have a Controller, Compute and a Gateway/Network node.  This is
>running Folsom.  Most of the services are up, except that I cannot ping
>the floating ip of the VM.  
>
>Below are the error messages I am seeing the Gateway/Network node:
>
>
>root@gateway:/var/log/quantum# ovs-vsctl show
>
>9f3f021a-cb8e-4ef4-87a5-f77775bab9f3
>
>    Bridge br-ex
>
>        Port "eth2"
>
>            Interface "eth2"
>
>        Port "qg-0aff855e-9c"
>
>            Interface "qg-0aff855e-9c"
>
>                type: internal
>
>        Port br-ex
>
>            Interface br-ex
>
>                type: internal
>
>    Bridge "br-eth1"
>
>        Port "eth1"
>
>            Interface "eth1"
>
>        Port "phy-br-eth1"
>
>            Interface "phy-br-eth1"
>
>        Port "br-eth1"
>
>            Interface "br-eth1"
>
>                type: internal
>
>    Bridge br-int
>
>        Port br-int
>
>            Interface br-int
>
>                type: internal
>
>        Port "int-br-eth1"
>
>            Interface "int-br-eth1"
>
>        Port "tap47ac110a-fe"
>
>            tag: 1
>
>            Interface "tap47ac110a-fe"
>
>                type: internal
>
>        Port "qr-12091c58-8a"
>
>            tag: 1
>
>            Interface "qr-12091c58-8a"
>
>                type: internal
>
>    ovs_version: "1.4.3"
>
>
>=====================================
>
>L3 Agent.log:
>
>
>2013-04-16 22:06:08    DEBUG [quantum.agent.linux.utils] Running
>command: ip -o link show br-ex
>
>2013-04-16 22:06:08    DEBUG [quantum.agent.linux.utils] 
>
>Command: ['ip', '-o', 'link', 'show', 'br-ex']
>
>Exit code: 0
>
>Stdout: '20: br-ex: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state
>DOWN mode DEFAULT \\    link/ether 00:04:23:c9:64:ac brd
>ff:ff:ff:ff:ff:ff\n'
>
>Stderr: ''
>
>2013-04-16 22:06:08    DEBUG [quantumclient.client] REQ: curl -i
>http://192.168.0.2:9696//v2.0/networks.json?router%3Aexternal=True -X
>GET -H "User-Agent: python-quantumclient" -H "Content-Type:
>application/json" -H "Accept: application/json" -H "X-Auth-Token:
>3c2cc4afb68f4042bf4bddd9908186d0"
>
>
>2013-04-16 22:06:08    DEBUG [quantumclient.client] RESP:{'date': 'Wed,
>17 Apr 2013 02:06:08 GMT', 'status': '200', 'content-length': '376',
>'content-type': 'application/json', 'content-location':
>u'http://192.168.0.2:9696//v2.0/networks.json?router%3Aexternal=True'}
>
>
>2013-04-16 22:06:08    DEBUG [quantumclient.client] RESP
>BODY:{"networks": [{"status": "ACTIVE", "subnets":
>["487ccc01-81ea-4abc-bbb4-061cde722801"], "name": "ext_net",
>"provider:physical_network": "physnet1", "admin_state_up": true,
>"tenant_id": "6bd416e48a394f06bfbf4d0856740262",
>"provider:network_type": "vlan", "router:external": true, "shared":
>false, "id": "2f2aa2c9-47be-4b3e-9f05-a547a8d3aaa9",
>"provider:segmentation_id": 500}]}
>
>
>2013-04-16 22:06:08    DEBUG [quantumclient.client] REQ: curl -i
>http://192.168.0.2:9696//v2.0/routers.json -X GET -H "User-Agent:
>python-quantumclient" -H "Content-Type: application/json" -H "Accept:
>application/json" -H "X-Auth-Token: 3c2cc4afb68f4042bf4bddd9908186d0"
>
>
>2013-04-16 22:06:08    DEBUG [quantumclient.client] RESP:{'date': 'Wed,
>17 Apr 2013 02:06:08 GMT', 'status': '200', 'content-length': '262',
>'content-type': 'application/json', 'content-location':
>u'http://192.168.0.2:9696//v2.0/routers.json'}
>
>
>2013-04-16 22:06:08    DEBUG [quantumclient.client] RESP
>BODY:{"routers": [{"status": "ACTIVE", "external_gateway_info":
>{"network_id": "2f2aa2c9-47be-4b3e-9f05-a547a8d3aaa9"}, "name":
>"router_proj_one", "admin_state_up": true, "tenant_id":
>"b443045cd0d54377b637cc0ba998bf89", "id":
>"49a92bbc-5f58-400a-a8cb-aebcf54db446"}]}
>
>
>2013-04-16 22:06:08    DEBUG [quantumclient.client] REQ: curl -i
>http://192.168.0.2:9696//v2.0/ports.json?device_owner=network%3Arouter_gateway&device_id=49a92bbc-5f58-400a-a8cb-aebcf54db446
>-X GET -H "User-Agent: python-quantumclient" -H "Content-Type:
>application/json" -H "Accept: application/json" -H "X-Auth-Token:
>3c2cc4afb68f4042bf4bddd9908186d0"
>
>
>2013-04-16 22:06:08    DEBUG [quantumclient.client] RESP:{'date': 'Wed,
>17 Apr 2013 02:06:08 GMT', 'status': '200', 'content-length': '417',
>'content-type': 'application/json', 'content-location':
>u'http://192.168.0.2:9696//v2.0/ports.json?device_owner=network%3Arouter_gateway&device_id=49a92bbc-5f58-400a-a8cb-aebcf54db446'}
>
>
>2013-04-16 22:06:08    DEBUG [quantumclient.client] RESP BODY:{"ports":
>[{"status": "ACTIVE", "name": "", "admin_state_up": true, "network_id":
>"2f2aa2c9-47be-4b3e-9f05-a547a8d3aaa9", "tenant_id": "",
>"device_owner": "network:router_gateway", "mac_address":
>"fa:16:3e:3a:fe:6a", "fixed_ips": [{"subnet_id":
>"487ccc01-81ea-4abc-bbb4-061cde722801", "ip_address": "192.168.1.10"}],
>"id": "0aff855e-9cab-4dad-a490-7019e104603a", "device_id":
>"49a92bbc-5f58-400a-a8cb-aebcf54db446"}]}
>
>
>2013-04-16 22:06:08    DEBUG [quantumclient.client] REQ: curl -i
>http://192.168.0.2:9696//v2.0/ports.json?device_owner=network%3Arouter_interface&device_id=49a92bbc-5f58-400a-a8cb-aebcf54db446
>-X GET -H "User-Agent: python-quantumclient" -H "Content-Type:
>application/json" -H "Accept: application/json" -H "X-Auth-Token:
>3c2cc4afb68f4042bf4bddd9908186d0"
>
>
>2013-04-16 22:06:08    DEBUG [quantumclient.client] RESP:{'date': 'Wed,
>17 Apr 2013 02:06:09 GMT', 'status': '200', 'content-length': '452',
>'content-type': 'application/json', 'content-location':
>u'http://192.168.0.2:9696//v2.0/ports.json?device_owner=network%3Arouter_interface&device_id=49a92bbc-5f58-400a-a8cb-aebcf54db446'}
>
>
>2013-04-16 22:06:08    DEBUG [quantumclient.client] RESP BODY:{"ports":
>[{"status": "ACTIVE", "name": "", "admin_state_up": true, "network_id":
>"53e1bcd2-d107-4236-aa3a-718bfa641ead", "tenant_id":
>"b443045cd0d54377b637cc0ba998bf89", "device_owner":
>"network:router_interface", "mac_address": "fa:16:3e:52:8b:d5",
>"fixed_ips": [{"subnet_id": "47e9644b-c536-4662-8771-2f296d47ad4d",
>"ip_address": "192.168.100.1"}], "id":
>"12091c58-8a04-4b0d-bf8a-53575e16298a", "device_id":
>"49a92bbc-5f58-400a-a8cb-aebcf54db446"}]}
>
>
>2013-04-16 22:06:08    DEBUG [quantumclient.client] REQ: curl -i
>http://192.168.0.2:9696//v2.0/floatingips.json?router_id=49a92bbc-5f58-400a-a8cb-aebcf54db446
>-X GET -H "User-Agent: python-quantumclient" -H "Content-Type:
>application/json" -H "Accept: application/json" -H "X-Auth-Token:
>3c2cc4afb68f4042bf4bddd9908186d0"
>
>
>2013-04-16 22:06:08    DEBUG [quantumclient.client] RESP:{'date': 'Wed,
>17 Apr 2013 02:06:09 GMT', 'status': '200', 'content-length': '357',
>'content-type': 'application/json', 'content-location':
>u'http://192.168.0.2:9696//v2.0/floatingips.json?router_id=49a92bbc-5f58-400a-a8cb-aebcf54db446'}
>
>
>2013-04-16 22:06:08    DEBUG [quantumclient.client] RESP
>BODY:{"floatingips": [{"router_id":
>"49a92bbc-5f58-400a-a8cb-aebcf54db446", "tenant_id":
>"b443045cd0d54377b637cc0ba998bf89", "floating_network_id":
>"2f2aa2c9-47be-4b3e-9f05-a547a8d3aaa9", "fixed_ip_address":
>"192.168.100.3", "floating_ip_address": "192.168.1.11", "port_id":
>"015cfbe5-9310-43cd-8edf-321d5907fa56", "id":
>"0613c05f-8ae4-4596-a0fa-35e72c96c6e4"}]}
>
>
>==============================================
>
>
>openvswitch-agent.log
>
>
>2013-04-16 22:06:50    DEBUG [quantum.agent.linux.utils] Running
>command: sudo /usr/bin/quantum-rootwrap /etc/quantum/rootwrap.conf
>ovs-vsctl --timeout=2 list-ports br-int
>
>2013-04-16 22:06:50    DEBUG [quantum.agent.linux.utils] 
>
>Command: ['sudo', '/usr/bin/quantum-rootwrap',
>'/etc/quantum/rootwrap.conf', 'ovs-vsctl', '--timeout=2', 'list-ports',
>'br-int']
>
>Exit code: 0
>
>Stdout: 'int-br-eth1\nqr-12091c58-8a\ntap47ac110a-fe\n'
>
>Stderr: ''
>
>2013-04-16 22:06:50    DEBUG [quantum.agent.linux.utils] Running
>command: sudo /usr/bin/quantum-rootwrap /etc/quantum/rootwrap.conf
>ovs-vsctl --timeout=2 get Interface int-br-eth1 external_ids
>
>2013-04-16 22:06:50    DEBUG [quantum.agent.linux.utils] 
>
>Command: ['sudo', '/usr/bin/quantum-rootwrap',
>'/etc/quantum/rootwrap.conf', 'ovs-vsctl', '--timeout=2', 'get',
>'Interface', 'int-br-eth1', 'external_ids']
>
>Exit code: 0
>
>Stdout: '{}\n'
>
>Stderr: ''
>
>2013-04-16 22:06:50    DEBUG [quantum.agent.linux.utils] Running
>command: sudo /usr/bin/quantum-rootwrap /etc/quantum/rootwrap.conf
>ovs-vsctl --timeout=2 get Interface qr-12091c58-8a external_ids
>
>2013-04-16 22:06:50    DEBUG [quantum.agent.linux.utils] 
>
>Command: ['sudo', '/usr/bin/quantum-rootwrap',
>'/etc/quantum/rootwrap.conf', 'ovs-vsctl', '--timeout=2', 'get',
>'Interface', 'qr-12091c58-8a', 'external_ids']
>
>Exit code: 0
>
>Stdout: '{attached-mac="fa:16:3e:52:8b:d5",
>iface-id="12091c58-8a04-4b0d-bf8a-53575e16298a",
>iface-status=active}\n'
>
>Stderr: ''
>
>2013-04-16 22:06:50    DEBUG [quantum.agent.linux.utils] Running
>command: sudo /usr/bin/quantum-rootwrap /etc/quantum/rootwrap.conf
>ovs-vsctl --timeout=2 get Interface tap47ac110a-fe external_ids
>
>2013-04-16 22:06:51    DEBUG [quantum.agent.linux.utils] 
>
>Command: ['sudo', '/usr/bin/quantum-rootwrap',
>'/etc/quantum/rootwrap.conf', 'ovs-vsctl', '--timeout=2', 'get',
>'Interface', 'tap47ac110a-fe', 'external_ids']
>
>Exit code: 0
>
>Stdout: '{attached-mac="fa:16:3e:02:f5:f9",
>iface-id="47ac110a-fe25-43ec-b83a-4db58071a3d3",
>iface-status=active}\n'
>
>Stderr: ''
>
>
>=============================================================
>
>
>root@gateway:/var/log/quantum# sudo /usr/bin/quantum-rootwrap
>/etc/quantum/rootwrap.conf ovs-vsctl --timeout=2 list-ports br-int
>
>int-br-eth1
>
>qr-12091c58-8a
>
>tap47ac110a-fe
>
>
>
>============================================================================
>
>
>
>ovs-vswitchd.log
>
>
>Apr 16 22:08:39|00482|netdev|WARN|Dropped 233 log messages in last 12
>seconds (most recently, 1 seconds ago) due to excessive rate
>
>Apr 16 22:08:39|00483|netdev|WARN|failed to get flags for network
>device qr-12091c58-8a: No such device
>
>Apr 16 22:08:43|00484|netdev_linux|WARN|Dropped 35 log messages in last
>15 seconds (most recently, 5 seconds ago) due to excessive rate
>
>Apr 16 22:08:43|00485|netdev_linux|WARN|ioctl(SIOCGIFINDEX) on
>qr-12091c58-8a device failed: No such device
>
>Apr 16 22:08:52|00486|netdev|WARN|Dropped 233 log messages in last 12
>seconds (most recently, 1 seconds ago) due to excessive rate
>
>Apr 16 22:08:52|00487|netdev|WARN|failed to get flags for network
>device qr-12091c58-8a: No such device
>
>Apr 16 22:08:53|00488|netdev_linux|WARN|Dropped 23 log messages in last
>10 seconds (most recently, 5 seconds ago) due to excessive rate
>
>Apr 16 22:08:53|00489|netdev_linux|WARN|ioctl(SIOCGIFINDEX) on
>qr-12091c58-8a device failed: No such device
>
>Apr 16 22:09:04|00490|netdev|WARN|Dropped 227 log messages in last 12
>seconds (most recently, 1 seconds ago) due to excessive rate
>
>Apr 16 22:09:04|00491|netdev|WARN|failed to get flags for network
>device qr-12091c58-8a: No such device
>
>
>=====================================
>
>
>oot@gateway:/var/log/quantum# ifconfig -a
>
>br-eth1   Link encap:Ethernet  HWaddr 00:13:72:55:86:33  
>
>          BROADCAST MULTICAST  MTU:1500  Metric:1
>
>          RX packets:728 errors:0 dropped:0 overruns:0 frame:0
>
>          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
>
>          collisions:0 txqueuelen:0 
>
>          RX bytes:172923 (172.9 KB)  TX bytes:0 (0.0 B)
>
>
>br-ex     Link encap:Ethernet  HWaddr 00:04:23:c9:64:ac  
>
>          BROADCAST MULTICAST  MTU:1500  Metric:1
>
>          RX packets:695 errors:0 dropped:0 overruns:0 frame:0
>
>          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
>
>          collisions:0 txqueuelen:0 
>
>          RX bytes:161089 (161.0 KB)  TX bytes:0 (0.0 B)
>
>
>br-int    Link encap:Ethernet  HWaddr 2a:e5:80:ce:aa:43  
>
>          BROADCAST MULTICAST  MTU:1500  Metric:1
>
>          RX packets:23 errors:0 dropped:0 overruns:0 frame:0
>
>          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
>
>          collisions:0 txqueuelen:0 
>
>          RX bytes:4496 (4.4 KB)  TX bytes:0 (0.0 B)
>
>
>eth0      Link encap:Ethernet  HWaddr 00:13:72:55:86:32  
>
>         inet addr:192.168.0.3  Bcast:192.168.0.255  Mask:255.255.255.0
>
>          inet6 addr: fe80::213:72ff:fe55:8632/64 Scope:Link
>
>          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
>
>          RX packets:5262 errors:0 dropped:0 overruns:0 frame:0
>
>          TX packets:3830 errors:0 dropped:0 overruns:0 carrier:0
>
>          collisions:0 txqueuelen:1000 
>
>          RX bytes:975352 (975.3 KB)  TX bytes:816388 (816.3 KB)
>
>
>eth1      Link encap:Ethernet  HWaddr 00:13:72:55:86:33  
>
>          inet addr:10.10.10.3  Bcast:10.10.10.255  Mask:255.255.255.0
>
>          inet6 addr: fe80::213:72ff:fe55:8633/64 Scope:Link
>
>          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
>
>          RX packets:1690 errors:0 dropped:0 overruns:0 frame:0
>
>          TX packets:56 errors:0 dropped:0 overruns:0 carrier:0
>
>          collisions:0 txqueuelen:1000 
>
>          RX bytes:327902 (327.9 KB)  TX bytes:8684 (8.6 KB)
>
>
>eth2      Link encap:Ethernet  HWaddr 00:04:23:c9:64:ac  
>
>         inet addr:192.168.1.5  Bcast:192.168.1.255  Mask:255.255.255.0
>
>          inet6 addr: fe80::204:23ff:fec9:64ac/64 Scope:Link
>
>          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
>
>          RX packets:1664 errors:0 dropped:0 overruns:0 frame:0
>
>          TX packets:423 errors:0 dropped:0 overruns:0 carrier:0
>
>          collisions:0 txqueuelen:1000 
>
>          RX bytes:318659 (318.6 KB)  TX bytes:26402 (26.4 KB)
>
>
>eth3      Link encap:Ethernet  HWaddr 00:04:23:c9:64:ad  
>
>          BROADCAST MULTICAST  MTU:1500  Metric:1
>
>          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
>
>          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
>
>          collisions:0 txqueuelen:1000 
>
>          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)
>
>
>int-br-eth1 Link encap:Ethernet  HWaddr 7e:42:53:c7:c4:f2  
>
>          inet6 addr: fe80::7c42:53ff:fec7:c4f2/64 Scope:Link
>
>          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
>
>          RX packets:698 errors:0 dropped:0 overruns:0 frame:0
>
>          TX packets:39 errors:0 dropped:0 overruns:0 carrier:0
>
>          collisions:0 txqueuelen:1000 
>
>          RX bytes:164330 (164.3 KB)  TX bytes:5955 (5.9 KB)
>
>
>lo        Link encap:Local Loopback  
>
>          inet addr:127.0.0.1  Mask:255.0.0.0
>
>          inet6 addr: ::1/128 Scope:Host
>
>          UP LOOPBACK RUNNING  MTU:16436  Metric:1
>
>          RX packets:194 errors:0 dropped:0 overruns:0 frame:0
>
>          TX packets:194 errors:0 dropped:0 overruns:0 carrier:0
>
>          collisions:0 txqueuelen:0 
>
>          RX bytes:18778 (18.7 KB)  TX bytes:18778 (18.7 KB)
>
>
>phy-br-eth1 Link encap:Ethernet  HWaddr 3a:a7:27:ec:28:4e  
>
>          inet6 addr: fe80::38a7:27ff:feec:284e/64 Scope:Link
>
>          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
>
>          RX packets:39 errors:0 dropped:0 overruns:0 frame:0
>
>          TX packets:698 errors:0 dropped:0 overruns:0 carrier:0
>
>          collisions:0 txqueuelen:1000 
>
>          RX bytes:5955 (5.9 KB)  TX bytes:164330 (164.3 KB)

-- 
Sent from my Android tablet.
References

[Doc] Talk shop: API Docs
From: Anne Gentle, 2013-04-16
Re: [Doc] Talk shop: API Docs
From: thingee, 2013-04-16
OpenStack Multi Node Problems with OVS
From: Atif Wasi, 2013-04-17