yahoo-eng-team team mailing list archive
-
yahoo-eng-team team
-
Mailing list archive
-
Message #54368
[Bug 1607398] [NEW] DVR, Floating IPs are not working. Failed sending gratuitous ARP
Public bug reported:
Hello, I'm trying to use DVR and floating IPs, but it does not work.
When I'm trying to associate a floating IP with VM, I see on the compute node:
2016-07-28 14:26:31.893 125513 DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'fip-4f2774d1-dfb8-4833-8374-806e1fc40827', 'arping', '-A', '-I', 'fg-86481da8-4c', '-c', '3', '-w', '4.5', '172.16.48.6'] execute_rootwrap_daemon /usr/lib/python2.7/site-packages/neutron/agent/linux/utils.py:100
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.utils [-] Exit code: 2; Stdin: ; Stdout: ; Stderr: bind: Cannot assign requested address
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib [-] Failed sending gratuitous ARP to 172.16.48.6 on fg-86481da8-4c in namespace fip-4f2774d1-dfb8-4833-8374-806e1fc40827
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib Traceback (most recent call last):
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib File "/usr/lib/python2.7/site-packages/neutron/agent/linux/ip_lib.py", line 1040, in _arping
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib ip_wrapper.netns.execute(arping_cmd, check_exit_code=True)
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib File "/usr/lib/python2.7/site-packages/neutron/agent/linux/ip_lib.py", line 927, in execute
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib log_fail_as_error=log_fail_as_error, **kwargs)
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib File "/usr/lib/python2.7/site-packages/neutron/agent/linux/utils.py", line 140, in execute
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib raise RuntimeError(msg)
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib RuntimeError: Exit code: 2; Stdin: ; Stdout: ; Stderr: bind: Cannot assign requested address
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib
2016-07-28 14:26:31.948 125513 DEBUG oslo_messaging._drivers.amqpdriver [-] received reply msg_id: 67908aabc9bd446493cd22af8cccbd59 __call__ /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py:302
2016-07-28 14:26:31.949 125513 DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'fip-4f2774d1-dfb8-4833-8374-806e1fc40827', 'ip', '-o', 'link', 'show', 'fpr-a5e261f2-9'] execute_rootwrap_daemon /usr/lib/python2.7/site-packages/neutron/agent/linux/utils.py:100
[root@node13 ~]# sysctl net.ipv4.ip_nonlocal_bind
net.ipv4.ip_nonlocal_bind = 1
[root@node13 ~]# ip netns exec fip-4f2774d1-dfb8-4833-8374-806e1fc40827 sysctl net.ipv4.ip_nonlocal_bind
net.ipv4.ip_nonlocal_bind = 1
Remote ping is not working:
[root@node13 ~]# ping -c2 172.16.48.6
PING 172.16.48.6 (172.16.48.6) 56(84) bytes of data.
^C
--- 172.16.48.6 ping statistics ---
2 packets transmitted, 0 received, 100% packet loss, time 999ms
Ping into the namespace is working:
[root@node13 ~]# ip netns
fip-4f2774d1-dfb8-4833-8374-806e1fc40827
qrouter-a5e261f2-991c-497c-adcd-b1e9e1a8a001
[root@node13 ~]# ip netns exec fip-4f2774d1-dfb8-4833-8374-806e1fc40827 ping -c2 172.16.48.6
PING 172.16.48.6 (172.16.48.6) 56(84) bytes of data.
64 bytes from 172.16.48.6: icmp_seq=1 ttl=63 time=0.290 ms
64 bytes from 172.16.48.6: icmp_seq=2 ttl=63 time=0.260 ms
--- 172.16.48.6 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1000ms
rtt min/avg/max/mdev = 0.260/0.275/0.290/0.015 ms
Additional information:
[root@node13 ~]# ip netns exec fip-4f2774d1-dfb8-4833-8374-806e1fc40827 ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
9: fpr-a5e261f2-9: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether ce:cd:c7:4d:b8:c2 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 169.254.126.51/31 scope global fpr-a5e261f2-9
valid_lft forever preferred_lft forever
inet6 fe80::cccd:c7ff:fe4d:b8c2/64 scope link
valid_lft forever preferred_lft forever
333: fg-86481da8-4c: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN
link/ether fa:16:3e:3b:ac:ac brd ff:ff:ff:ff:ff:ff
inet 172.16.48.4/22 brd 172.16.51.255 scope global fg-86481da8-4c
valid_lft forever preferred_lft forever
inet6 fe80::f816:3eff:fe3b:acac/64 scope link
valid_lft forever preferred_lft forever
[root@node13 ~]# ip netns exec fip-4f2774d1-dfb8-4833-8374-806e1fc40827 ip route
default via 172.16.51.254 dev fg-86481da8-4c
169.254.126.50/31 dev fpr-a5e261f2-9 proto kernel scope link src 169.254.126.51
172.16.48.0/22 dev fg-86481da8-4c proto kernel scope link src 172.16.48.4
172.16.48.6 via 169.254.126.50 dev fpr-a5e261f2-9
[root@node13 ~]# ovs-vsctl show
8fd67521-01bc-4f6e-8750-a2501d4e1505
Bridge br-int
fail_mode: secure
Port "fg-86481da8-4c"
tag: 2
Interface "fg-86481da8-4c"
type: internal
Port br-int
Interface br-int
type: internal
Port patch-tun
Interface patch-tun
type: patch
options: {peer=patch-int}
Port "tapf872f999-f9"
tag: 4095
Interface "tapf872f999-f9"
type: internal
Port "qvo616a1035-c5"
tag: 1
Interface "qvo616a1035-c5"
Port int-br-ex
Interface int-br-ex
type: patch
options: {peer=phy-br-ex}
Port int-br-private
Interface int-br-private
type: patch
options: {peer=phy-br-private}
Port "qr-9edb3525-5a"
tag: 1
Interface "qr-9edb3525-5a"
type: internal
Bridge br-tun
fail_mode: secure
Port "vxlan-ac103409"
Interface "vxlan-ac103409"
type: vxlan
options: {df_default="true", in_key=flow, local_ip="172.16.52.23", out_key=flow, remote_ip="172.16.52.9"}
Port br-tun
Interface br-tun
type: internal
Port "vxlan-ac103418"
Interface "vxlan-ac103418"
type: vxlan
options: {df_default="true", in_key=flow, local_ip="172.16.52.23", out_key=flow, remote_ip="172.16.52.24"}
Port "vxlan-ac103403"
Interface "vxlan-ac103403"
type: vxlan
options: {df_default="true", in_key=flow, local_ip="172.16.52.23", out_key=flow, remote_ip="172.16.52.3"}
Port patch-int
Interface patch-int
type: patch
options: {peer=patch-tun}
Bridge br-ex
Port br-ex
Interface br-ex
type: internal
Port "em1.193"
Interface "em1.193"
Port phy-br-ex
Interface phy-br-ex
type: patch
options: {peer=int-br-ex}
Bridge br-private
Port phy-br-private
Interface phy-br-private
type: patch
options: {peer=int-br-private}
Port br-private
Interface br-private
type: internal
Port "em1.1070"
Interface "em1.1070"
ovs_version: "2.5.0"
** Affects: neutron
Importance: Undecided
Status: New
** Tags: dvr floating ips
--
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to neutron.
https://bugs.launchpad.net/bugs/1607398
Title:
DVR, Floating IPs are not working. Failed sending gratuitous ARP
Status in neutron:
New
Bug description:
Hello, I'm trying to use DVR and floating IPs, but it does not work.
When I'm trying to associate a floating IP with VM, I see on the compute node:
2016-07-28 14:26:31.893 125513 DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'fip-4f2774d1-dfb8-4833-8374-806e1fc40827', 'arping', '-A', '-I', 'fg-86481da8-4c', '-c', '3', '-w', '4.5', '172.16.48.6'] execute_rootwrap_daemon /usr/lib/python2.7/site-packages/neutron/agent/linux/utils.py:100
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.utils [-] Exit code: 2; Stdin: ; Stdout: ; Stderr: bind: Cannot assign requested address
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib [-] Failed sending gratuitous ARP to 172.16.48.6 on fg-86481da8-4c in namespace fip-4f2774d1-dfb8-4833-8374-806e1fc40827
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib Traceback (most recent call last):
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib File "/usr/lib/python2.7/site-packages/neutron/agent/linux/ip_lib.py", line 1040, in _arping
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib ip_wrapper.netns.execute(arping_cmd, check_exit_code=True)
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib File "/usr/lib/python2.7/site-packages/neutron/agent/linux/ip_lib.py", line 927, in execute
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib log_fail_as_error=log_fail_as_error, **kwargs)
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib File "/usr/lib/python2.7/site-packages/neutron/agent/linux/utils.py", line 140, in execute
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib raise RuntimeError(msg)
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib RuntimeError: Exit code: 2; Stdin: ; Stdout: ; Stderr: bind: Cannot assign requested address
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib
2016-07-28 14:26:31.912 125513 ERROR neutron.agent.linux.ip_lib
2016-07-28 14:26:31.948 125513 DEBUG oslo_messaging._drivers.amqpdriver [-] received reply msg_id: 67908aabc9bd446493cd22af8cccbd59 __call__ /usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py:302
2016-07-28 14:26:31.949 125513 DEBUG neutron.agent.linux.utils [-] Running command (rootwrap daemon): ['ip', 'netns', 'exec', 'fip-4f2774d1-dfb8-4833-8374-806e1fc40827', 'ip', '-o', 'link', 'show', 'fpr-a5e261f2-9'] execute_rootwrap_daemon /usr/lib/python2.7/site-packages/neutron/agent/linux/utils.py:100
[root@node13 ~]# sysctl net.ipv4.ip_nonlocal_bind
net.ipv4.ip_nonlocal_bind = 1
[root@node13 ~]# ip netns exec fip-4f2774d1-dfb8-4833-8374-806e1fc40827 sysctl net.ipv4.ip_nonlocal_bind
net.ipv4.ip_nonlocal_bind = 1
Remote ping is not working:
[root@node13 ~]# ping -c2 172.16.48.6
PING 172.16.48.6 (172.16.48.6) 56(84) bytes of data.
^C
--- 172.16.48.6 ping statistics ---
2 packets transmitted, 0 received, 100% packet loss, time 999ms
Ping into the namespace is working:
[root@node13 ~]# ip netns
fip-4f2774d1-dfb8-4833-8374-806e1fc40827
qrouter-a5e261f2-991c-497c-adcd-b1e9e1a8a001
[root@node13 ~]# ip netns exec fip-4f2774d1-dfb8-4833-8374-806e1fc40827 ping -c2 172.16.48.6
PING 172.16.48.6 (172.16.48.6) 56(84) bytes of data.
64 bytes from 172.16.48.6: icmp_seq=1 ttl=63 time=0.290 ms
64 bytes from 172.16.48.6: icmp_seq=2 ttl=63 time=0.260 ms
--- 172.16.48.6 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1000ms
rtt min/avg/max/mdev = 0.260/0.275/0.290/0.015 ms
Additional information:
[root@node13 ~]# ip netns exec fip-4f2774d1-dfb8-4833-8374-806e1fc40827 ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
9: fpr-a5e261f2-9: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether ce:cd:c7:4d:b8:c2 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 169.254.126.51/31 scope global fpr-a5e261f2-9
valid_lft forever preferred_lft forever
inet6 fe80::cccd:c7ff:fe4d:b8c2/64 scope link
valid_lft forever preferred_lft forever
333: fg-86481da8-4c: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN
link/ether fa:16:3e:3b:ac:ac brd ff:ff:ff:ff:ff:ff
inet 172.16.48.4/22 brd 172.16.51.255 scope global fg-86481da8-4c
valid_lft forever preferred_lft forever
inet6 fe80::f816:3eff:fe3b:acac/64 scope link
valid_lft forever preferred_lft forever
[root@node13 ~]# ip netns exec fip-4f2774d1-dfb8-4833-8374-806e1fc40827 ip route
default via 172.16.51.254 dev fg-86481da8-4c
169.254.126.50/31 dev fpr-a5e261f2-9 proto kernel scope link src 169.254.126.51
172.16.48.0/22 dev fg-86481da8-4c proto kernel scope link src 172.16.48.4
172.16.48.6 via 169.254.126.50 dev fpr-a5e261f2-9
[root@node13 ~]# ovs-vsctl show
8fd67521-01bc-4f6e-8750-a2501d4e1505
Bridge br-int
fail_mode: secure
Port "fg-86481da8-4c"
tag: 2
Interface "fg-86481da8-4c"
type: internal
Port br-int
Interface br-int
type: internal
Port patch-tun
Interface patch-tun
type: patch
options: {peer=patch-int}
Port "tapf872f999-f9"
tag: 4095
Interface "tapf872f999-f9"
type: internal
Port "qvo616a1035-c5"
tag: 1
Interface "qvo616a1035-c5"
Port int-br-ex
Interface int-br-ex
type: patch
options: {peer=phy-br-ex}
Port int-br-private
Interface int-br-private
type: patch
options: {peer=phy-br-private}
Port "qr-9edb3525-5a"
tag: 1
Interface "qr-9edb3525-5a"
type: internal
Bridge br-tun
fail_mode: secure
Port "vxlan-ac103409"
Interface "vxlan-ac103409"
type: vxlan
options: {df_default="true", in_key=flow, local_ip="172.16.52.23", out_key=flow, remote_ip="172.16.52.9"}
Port br-tun
Interface br-tun
type: internal
Port "vxlan-ac103418"
Interface "vxlan-ac103418"
type: vxlan
options: {df_default="true", in_key=flow, local_ip="172.16.52.23", out_key=flow, remote_ip="172.16.52.24"}
Port "vxlan-ac103403"
Interface "vxlan-ac103403"
type: vxlan
options: {df_default="true", in_key=flow, local_ip="172.16.52.23", out_key=flow, remote_ip="172.16.52.3"}
Port patch-int
Interface patch-int
type: patch
options: {peer=patch-tun}
Bridge br-ex
Port br-ex
Interface br-ex
type: internal
Port "em1.193"
Interface "em1.193"
Port phy-br-ex
Interface phy-br-ex
type: patch
options: {peer=int-br-ex}
Bridge br-private
Port phy-br-private
Interface phy-br-private
type: patch
options: {peer=int-br-private}
Port br-private
Interface br-private
type: internal
Port "em1.1070"
Interface "em1.1070"
ovs_version: "2.5.0"
To manage notifications about this bug go to:
https://bugs.launchpad.net/neutron/+bug/1607398/+subscriptions
Follow ups
