← Back to team overview

yahoo-eng-team team mailing list archive

  1. yahoo-eng-team team
  2. Mailing list archive
  3. Message #81322

[Bug 1859887] [NEW] External connectivity broken because of stale FIP rule

  Thread Previous • Date Previous • • Thread Next 
Public bug reported:

Seen a few occurrences of this issue where I have a VM that does not
have a FIP attached, but has a port on a tenant network that is attached
to an external network via a router. I expect the VM to be able to reach
out to the external network, but I see nothing going through.

On the VM:
--snip--
[root@bob-trove-1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc pfifo_fast state UP qlen 1000
    link/ether fa:16:3e:97:b3:3b brd ff:ff:ff:ff:ff:ff
    inet 172.20.7.16/24 brd 172.20.7.255 scope global dynamic eth0
       valid_lft 68868sec preferred_lft 68868sec
[root@bob-trove-1 ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         172.20.7.1      0.0.0.0         UG    100    0        0 eth0
169.254.169.254 172.20.7.1      255.255.255.255 UGH   100    0        0 eth0
172.20.2.192    0.0.0.0         255.255.255.192 U     100    0        0 eth0
172.20.5.192    0.0.0.0         255.255.255.192 U     100    0        0 eth0
172.20.6.0      0.0.0.0         255.255.255.192 U     100    0        0 eth0
172.20.6.64     0.0.0.0         255.255.255.192 U     100    0        0 eth0
172.20.7.0      0.0.0.0         255.255.255.0   U     100    0        0 eth0
--snip--

>From the router namespace:
--snip--
root@kvm02:/# ip netns exec qrouter-ea187315-b0c7-4f2e-98e9-128a923fca4e ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: rfp-ea187315-b@if292: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 4e:54:d8:b1:6a:6d brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 169.254.114.242/31 scope global rfp-ea187315-b
       valid_lft forever preferred_lft forever
    inet6 fe80::4c54:d8ff:feb1:6a6d/64 scope link
       valid_lft forever preferred_lft forever
15636: qr-81061dca-85: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ether fa:16:3e:94:27:37 brd ff:ff:ff:ff:ff:ff
    inet 192.0.3.1/24 brd 192.0.3.255 scope global qr-81061dca-85
       valid_lft forever preferred_lft forever
    inet6 fe80::f816:3eff:fe94:2737/64 scope link
       valid_lft forever preferred_lft forever
15703: qr-41aba180-7f: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ether fa:16:3e:a5:64:9c brd ff:ff:ff:ff:ff:ff
    inet 172.20.7.1/24 brd 172.20.7.255 scope global qr-41aba180-7f
       valid_lft forever preferred_lft forever
    inet6 fe80::f816:3eff:fea5:649c/64 scope link
       valid_lft forever preferred_lft forever
13957: qr-1408b658-c8: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ether fa:16:3e:ac:80:c4 brd ff:ff:ff:ff:ff:ff
    inet 172.20.6.1/26 brd 172.20.6.63 scope global qr-1408b658-c8
       valid_lft forever preferred_lft forever
    inet6 fe80::f816:3eff:feac:80c4/64 scope link
       valid_lft forever preferred_lft forever
11146: qr-127e45c0-8d: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ether fa:16:3e:82:03:97 brd ff:ff:ff:ff:ff:ff
    inet 172.20.5.193/26 brd 172.20.5.255 scope global qr-127e45c0-8d
       valid_lft forever preferred_lft forever
    inet6 fe80::f816:3eff:fe82:397/64 scope link
       valid_lft forever preferred_lft forever
11147: qr-3ebb2a27-9a: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ether fa:16:3e:cc:b9:95 brd ff:ff:ff:ff:ff:ff
    inet 172.20.2.193/26 brd 172.20.2.255 scope global qr-3ebb2a27-9a
       valid_lft forever preferred_lft forever
    inet6 fe80::f816:3eff:fecc:b995/64 scope link
       valid_lft forever preferred_lft forever
13970: qr-35480bae-20: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ether fa:16:3e:23:89:f3 brd ff:ff:ff:ff:ff:ff
    inet 172.20.6.65/26 brd 172.20.6.127 scope global qr-35480bae-20
       valid_lft forever preferred_lft forever
    inet6 fe80::f816:3eff:fe23:89f3/64 scope link
       valid_lft forever preferred_lft forever
root@kvm02:/# ip netns exec qrouter-ea187315-b0c7-4f2e-98e9-128a923fca4e ip rule
0:      from all lookup local
32766:  from all lookup main
32767:  from all lookup default
36707:  from 172.20.7.5 lookup 16
36709:  from 172.20.2.248 lookup 16
37304:  from 172.20.7.56 lookup 16
46130:  from 172.20.7.36 lookup 16
46133:  from 172.20.5.223 lookup 16
46134:  from 172.20.2.217 lookup 16
46138:  from 172.20.2.245 lookup 16
54173:  from 172.20.7.16 lookup 16
57482:  from 172.20.5.252 lookup 16
62083:  from 172.20.7.76 lookup 16
72399:  from 172.20.7.80 lookup 16
72454:  from 172.20.7.37 lookup 16
2886992577:     from 172.20.2.193/26 lookup 2886992577
2886993345:     from 172.20.5.193/26 lookup 2886993345
2886993409:     from 172.20.6.1/26 lookup 2886993409
2886993473:     from 172.20.6.65/26 lookup 2886993473
2886993665:     from 172.20.7.1/24 lookup 2886993665
3221226009:     from 192.0.2.25/24 lookup 3221226009
3221226241:     from 192.0.3.1/24 lookup 3221226241
root@kvm02:/# ip netns exec qrouter-ea187315-b0c7-4f2e-98e9-128a923fca4e ip route show table 16
default via 169.254.114.243 dev rfp-ea187315-b
root@kvm02:/#
--snip--

The VM does not have a FIP attached, but the router namespace has a rule
(54173:  from 172.20.7.16 lookup 16) that forwards traffic to the FIP
namespace.

Attaching a FIP gets the traffic flowing, but removing it puts it back
in this state. The only way to recover is to delete this ip rule
manually.

** Affects: neutron
     Importance: Undecided
         Status: New

-- 
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to neutron.
https://bugs.launchpad.net/bugs/1859887

Title:
  External connectivity broken because of stale FIP rule

Status in neutron:
  New

Bug description:
  Seen a few occurrences of this issue where I have a VM that does not
  have a FIP attached, but has a port on a tenant network that is
  attached to an external network via a router. I expect the VM to be
  able to reach out to the external network, but I see nothing going
  through.

  On the VM:
  --snip--
  [root@bob-trove-1 ~]# ip a
  1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
      link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
      inet 127.0.0.1/8 scope host lo
         valid_lft forever preferred_lft forever
  2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc pfifo_fast state UP qlen 1000
      link/ether fa:16:3e:97:b3:3b brd ff:ff:ff:ff:ff:ff
      inet 172.20.7.16/24 brd 172.20.7.255 scope global dynamic eth0
         valid_lft 68868sec preferred_lft 68868sec
  [root@bob-trove-1 ~]# route -n
  Kernel IP routing table
  Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
  0.0.0.0         172.20.7.1      0.0.0.0         UG    100    0        0 eth0
  169.254.169.254 172.20.7.1      255.255.255.255 UGH   100    0        0 eth0
  172.20.2.192    0.0.0.0         255.255.255.192 U     100    0        0 eth0
  172.20.5.192    0.0.0.0         255.255.255.192 U     100    0        0 eth0
  172.20.6.0      0.0.0.0         255.255.255.192 U     100    0        0 eth0
  172.20.6.64     0.0.0.0         255.255.255.192 U     100    0        0 eth0
  172.20.7.0      0.0.0.0         255.255.255.0   U     100    0        0 eth0
  --snip--

  From the router namespace:
  --snip--
  root@kvm02:/# ip netns exec qrouter-ea187315-b0c7-4f2e-98e9-128a923fca4e ip a
  1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
      link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
      inet 127.0.0.1/8 scope host lo
         valid_lft forever preferred_lft forever
      inet6 ::1/128 scope host
         valid_lft forever preferred_lft forever
  2: rfp-ea187315-b@if292: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
      link/ether 4e:54:d8:b1:6a:6d brd ff:ff:ff:ff:ff:ff link-netnsid 0
      inet 169.254.114.242/31 scope global rfp-ea187315-b
         valid_lft forever preferred_lft forever
      inet6 fe80::4c54:d8ff:feb1:6a6d/64 scope link
         valid_lft forever preferred_lft forever
  15636: qr-81061dca-85: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN group default qlen 1000
      link/ether fa:16:3e:94:27:37 brd ff:ff:ff:ff:ff:ff
      inet 192.0.3.1/24 brd 192.0.3.255 scope global qr-81061dca-85
         valid_lft forever preferred_lft forever
      inet6 fe80::f816:3eff:fe94:2737/64 scope link
         valid_lft forever preferred_lft forever
  15703: qr-41aba180-7f: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN group default qlen 1000
      link/ether fa:16:3e:a5:64:9c brd ff:ff:ff:ff:ff:ff
      inet 172.20.7.1/24 brd 172.20.7.255 scope global qr-41aba180-7f
         valid_lft forever preferred_lft forever
      inet6 fe80::f816:3eff:fea5:649c/64 scope link
         valid_lft forever preferred_lft forever
  13957: qr-1408b658-c8: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN group default qlen 1000
      link/ether fa:16:3e:ac:80:c4 brd ff:ff:ff:ff:ff:ff
      inet 172.20.6.1/26 brd 172.20.6.63 scope global qr-1408b658-c8
         valid_lft forever preferred_lft forever
      inet6 fe80::f816:3eff:feac:80c4/64 scope link
         valid_lft forever preferred_lft forever
  11146: qr-127e45c0-8d: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN group default qlen 1000
      link/ether fa:16:3e:82:03:97 brd ff:ff:ff:ff:ff:ff
      inet 172.20.5.193/26 brd 172.20.5.255 scope global qr-127e45c0-8d
         valid_lft forever preferred_lft forever
      inet6 fe80::f816:3eff:fe82:397/64 scope link
         valid_lft forever preferred_lft forever
  11147: qr-3ebb2a27-9a: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN group default qlen 1000
      link/ether fa:16:3e:cc:b9:95 brd ff:ff:ff:ff:ff:ff
      inet 172.20.2.193/26 brd 172.20.2.255 scope global qr-3ebb2a27-9a
         valid_lft forever preferred_lft forever
      inet6 fe80::f816:3eff:fecc:b995/64 scope link
         valid_lft forever preferred_lft forever
  13970: qr-35480bae-20: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN group default qlen 1000
      link/ether fa:16:3e:23:89:f3 brd ff:ff:ff:ff:ff:ff
      inet 172.20.6.65/26 brd 172.20.6.127 scope global qr-35480bae-20
         valid_lft forever preferred_lft forever
      inet6 fe80::f816:3eff:fe23:89f3/64 scope link
         valid_lft forever preferred_lft forever
  root@kvm02:/# ip netns exec qrouter-ea187315-b0c7-4f2e-98e9-128a923fca4e ip rule
  0:      from all lookup local
  32766:  from all lookup main
  32767:  from all lookup default
  36707:  from 172.20.7.5 lookup 16
  36709:  from 172.20.2.248 lookup 16
  37304:  from 172.20.7.56 lookup 16
  46130:  from 172.20.7.36 lookup 16
  46133:  from 172.20.5.223 lookup 16
  46134:  from 172.20.2.217 lookup 16
  46138:  from 172.20.2.245 lookup 16
  54173:  from 172.20.7.16 lookup 16
  57482:  from 172.20.5.252 lookup 16
  62083:  from 172.20.7.76 lookup 16
  72399:  from 172.20.7.80 lookup 16
  72454:  from 172.20.7.37 lookup 16
  2886992577:     from 172.20.2.193/26 lookup 2886992577
  2886993345:     from 172.20.5.193/26 lookup 2886993345
  2886993409:     from 172.20.6.1/26 lookup 2886993409
  2886993473:     from 172.20.6.65/26 lookup 2886993473
  2886993665:     from 172.20.7.1/24 lookup 2886993665
  3221226009:     from 192.0.2.25/24 lookup 3221226009
  3221226241:     from 192.0.3.1/24 lookup 3221226241
  root@kvm02:/# ip netns exec qrouter-ea187315-b0c7-4f2e-98e9-128a923fca4e ip route show table 16
  default via 169.254.114.243 dev rfp-ea187315-b
  root@kvm02:/#
  --snip--

  The VM does not have a FIP attached, but the router namespace has a
  rule (54173:  from 172.20.7.16 lookup 16) that forwards traffic to the
  FIP namespace.

  Attaching a FIP gets the traffic flowing, but removing it puts it back
  in this state. The only way to recover is to delete this ip rule
  manually.

To manage notifications about this bug go to:
https://bugs.launchpad.net/neutron/+bug/1859887/+subscriptions
  Thread Previous • Date Previous • • Thread Next