yahoo-eng-team team mailing list archive

Thread
Date
[Bug 1715395] [NEW] FWaaS: Firewall creation fails in case of distributed routers (Pike)

To: yahoo-eng-team@xxxxxxxxxxxxxxxxxxx
From: Jens Offenbach <wolle5050@xxxxxx>
Date: Wed, 06 Sep 2017 13:43:59 -0000
Reply-to: Bug 1715395 <1715395@xxxxxxxxxxxxxxxxxx>
Sender: bounces@xxxxxxxxxxxxx
Public bug reported:

I have manually setup a fresh OpenStack Pike HA environment based on
Ubuntu 16.04.3 in conjunction with DVR. Firewall creation works in case
of centralized routers, but when a firewall gets attached to a
distributed router, the firewall gets stuck in "PENDUNG UPDATE". The log
file contains the following exception:

2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server [req-28e7a23e-fa55-4358-9977-c1db08435624 dddfba8e02f746799a6408a523e6cd25 ed2d2efd86dd40e7a45491d8502318d3 - - -] Exception during message handling: AttributeError: 'DvrEdgeHaRouter' object has no attribute 'dist_fip_count'
2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server Traceback (most recent call last):
2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server   File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/server.py", line 160, in _process_incoming
2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server     res = self.dispatcher.dispatch(message)
2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server   File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 213, in dispatch
2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server     return self._do_dispatch(endpoint, method, ctxt, args)
2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server   File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 183, in _do_dispatch
2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server     result = func(ctxt, **new_args)
2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server   File "/usr/lib/python2.7/dist-packages/oslo_log/helpers.py", line 67, in wrapper
2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server     return method(*args, **kwargs)
2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server   File "/usr/lib/python2.7/dist-packages/neutron_fwaas/services/firewall/agents/l3reference/firewall_l3_agent.py", line 284, in create_firewall
2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server     firewall)
2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server   File "/usr/lib/python2.7/dist-packages/neutron_fwaas/services/firewall/drivers/linux/iptables_fwaas.py", line 89, in create_firewall
2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server     self._setup_firewall(agent_mode, apply_list, firewall)
2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server   File "/usr/lib/python2.7/dist-packages/neutron_fwaas/services/firewall/drivers/linux/iptables_fwaas.py", line 195, in _setup_firewall
2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server     agent_mode, router_info)
2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server   File "/usr/lib/python2.7/dist-packages/neutron_fwaas/services/firewall/drivers/linux/iptables_fwaas.py", line 119, in _get_ipt_mgrs_with_if_prefix
2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server     if router_info.dist_fip_count:
2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server AttributeError: 'DvrEdgeHaRouter' object has no attribute 'dist_fip_count'

Some version information:
$ pip list | grep neutron
neutron (11.0.0)
neutron-fwaas (11.0.0)
neutron-fwaas-dashboard (1.0.1.dev1)
neutron-lbaas (11.0.0)
neutron-lbaas-dashboard (3.0.1)
neutron-lib (1.9.1)

##############################
l3_agent.ini
##############################

[DEFAULT]
agent_mode = dvr_snat
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver

[agent]
extensions = fwaas

[fwaas]
agent_version = v1
driver = iptables
enabled = true

##############################
neutron.conf
##############################

[DEFAULT]
allow_overlapping_ips = true
auth_strategy = keystone
base_mac = 02:05:69:00:00:00
bind_host = 10.30.200.101
bind_port = 9696
core_plugin = ml2
debug = false
default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=WARN,oslo.messaging=WARN,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,requests.packages.urllib3.util.retry=WARN,urllib3.util.retry=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN,taskflow=WARN,keystoneauth=WARN,oslo.cache=WARN,dogpile.core.dogpile=WARN,oslo_service=WARN,neutron=WARN
dhcp_agents_per_network = 2
dns_domain = openstack.mycompany.com.
dvr_base_mac = 0A:05:69:00:00:00
endpoint_type = internalURL
host = os-network01
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
l3_ha = true
l3_ha_net_cidr = 169.254.192.0/18
log_dir = /var/log/neutron
max_l3_agents_per_router = 2
min_l3_agents_per_router = 2
notify_nova_on_port_data_changes = true
notify_nova_on_port_status_changes = true
router_distributed = true
service_plugins = router,firewall,qos,lbaasv2
state_path = /var/lib/neutron
transport_url = rabbit://neutron:neutronpass@os-rabbit01:5672,neutron:neutronpass@os-rabbit02:5672/openstack

[agent]
root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf

[database]
connection = mysql+pymysql://neutron:neutronDBpass@os-controller/neutron
max_retries = -1

[keystone_authtoken]
auth_type = password
auth_uri = https://os-cloud.mycompany.com:5000
auth_url = http://os-identity:35357
memcached_servers = os-memcache:11211
password = neutronpass
project_domain_name = default
project_name = service
user_domain_name = default
username = neutron

[nova]
auth_type = password
auth_url = http://os-identity:35357
endpoint_type = internal
password = novapass
project_domain_name = default
project_name = service
region_name = RegionOne
user_domain_name = default
username = nova

[oslo_concurrency]
lock_path = /var/lock/neutron

[oslo_messaging_notifications]
driver = messagingv2

[oslo_messaging_rabbit]
amqp_durable_queues = true
rabbit_ha_queues = true
rabbit_retry_backoff = 2
rabbit_retry_interval = 1

[oslo_middleware]
enable_proxy_headers_parsing = true

[service_providers]
service_provider = FIREWALL:Iptables:neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver:default
service_provider = LOADBALANCERV2:Haproxy:neutron_lbaas.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default

##############################
fwaas_driver.ini
##############################

[fwaas]
enabled = true
driver = neutron_fwaas.services.firewall.drivers.linux.iptables_fwaas.IptablesFwaasDriver

May someone please have a look.

** Affects: neutron
     Importance: Undecided
         Status: New


** Tags: fwaas

-- 
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to neutron.
https://bugs.launchpad.net/bugs/1715395

Title:
  FWaaS: Firewall creation fails in case of distributed routers (Pike)

Status in neutron:
  New

Bug description:
  I have manually setup a fresh OpenStack Pike HA environment based on
  Ubuntu 16.04.3 in conjunction with DVR. Firewall creation works in
  case of centralized routers, but when a firewall gets attached to a
  distributed router, the firewall gets stuck in "PENDUNG UPDATE". The
  log file contains the following exception:

  2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server [req-28e7a23e-fa55-4358-9977-c1db08435624 dddfba8e02f746799a6408a523e6cd25 ed2d2efd86dd40e7a45491d8502318d3 - - -] Exception during message handling: AttributeError: 'DvrEdgeHaRouter' object has no attribute 'dist_fip_count'
  2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server Traceback (most recent call last):
  2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server   File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/server.py", line 160, in _process_incoming
  2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server     res = self.dispatcher.dispatch(message)
  2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server   File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 213, in dispatch
  2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server     return self._do_dispatch(endpoint, method, ctxt, args)
  2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server   File "/usr/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 183, in _do_dispatch
  2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server     result = func(ctxt, **new_args)
  2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server   File "/usr/lib/python2.7/dist-packages/oslo_log/helpers.py", line 67, in wrapper
  2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server     return method(*args, **kwargs)
  2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server   File "/usr/lib/python2.7/dist-packages/neutron_fwaas/services/firewall/agents/l3reference/firewall_l3_agent.py", line 284, in create_firewall
  2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server     firewall)
  2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server   File "/usr/lib/python2.7/dist-packages/neutron_fwaas/services/firewall/drivers/linux/iptables_fwaas.py", line 89, in create_firewall
  2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server     self._setup_firewall(agent_mode, apply_list, firewall)
  2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server   File "/usr/lib/python2.7/dist-packages/neutron_fwaas/services/firewall/drivers/linux/iptables_fwaas.py", line 195, in _setup_firewall
  2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server     agent_mode, router_info)
  2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server   File "/usr/lib/python2.7/dist-packages/neutron_fwaas/services/firewall/drivers/linux/iptables_fwaas.py", line 119, in _get_ipt_mgrs_with_if_prefix
  2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server     if router_info.dist_fip_count:
  2017-09-06 13:58:29.572 22581 ERROR oslo_messaging.rpc.server AttributeError: 'DvrEdgeHaRouter' object has no attribute 'dist_fip_count'

  Some version information:
  $ pip list | grep neutron
  neutron (11.0.0)
  neutron-fwaas (11.0.0)
  neutron-fwaas-dashboard (1.0.1.dev1)
  neutron-lbaas (11.0.0)
  neutron-lbaas-dashboard (3.0.1)
  neutron-lib (1.9.1)

  ##############################
  l3_agent.ini
  ##############################

  [DEFAULT]
  agent_mode = dvr_snat
  interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver

  [agent]
  extensions = fwaas

  [fwaas]
  agent_version = v1
  driver = iptables
  enabled = true

  ##############################
  neutron.conf
  ##############################

  [DEFAULT]
  allow_overlapping_ips = true
  auth_strategy = keystone
  base_mac = 02:05:69:00:00:00
  bind_host = 10.30.200.101
  bind_port = 9696
  core_plugin = ml2
  debug = false
  default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=WARN,oslo.messaging=WARN,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,requests.packages.urllib3.util.retry=WARN,urllib3.util.retry=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN,taskflow=WARN,keystoneauth=WARN,oslo.cache=WARN,dogpile.core.dogpile=WARN,oslo_service=WARN,neutron=WARN
  dhcp_agents_per_network = 2
  dns_domain = openstack.mycompany.com.
  dvr_base_mac = 0A:05:69:00:00:00
  endpoint_type = internalURL
  host = os-network01
  interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
  l3_ha = true
  l3_ha_net_cidr = 169.254.192.0/18
  log_dir = /var/log/neutron
  max_l3_agents_per_router = 2
  min_l3_agents_per_router = 2
  notify_nova_on_port_data_changes = true
  notify_nova_on_port_status_changes = true
  router_distributed = true
  service_plugins = router,firewall,qos,lbaasv2
  state_path = /var/lib/neutron
  transport_url = rabbit://neutron:neutronpass@os-rabbit01:5672,neutron:neutronpass@os-rabbit02:5672/openstack

  [agent]
  root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf

  [database]
  connection = mysql+pymysql://neutron:neutronDBpass@os-controller/neutron
  max_retries = -1

  [keystone_authtoken]
  auth_type = password
  auth_uri = https://os-cloud.mycompany.com:5000
  auth_url = http://os-identity:35357
  memcached_servers = os-memcache:11211
  password = neutronpass
  project_domain_name = default
  project_name = service
  user_domain_name = default
  username = neutron

  [nova]
  auth_type = password
  auth_url = http://os-identity:35357
  endpoint_type = internal
  password = novapass
  project_domain_name = default
  project_name = service
  region_name = RegionOne
  user_domain_name = default
  username = nova

  [oslo_concurrency]
  lock_path = /var/lock/neutron

  [oslo_messaging_notifications]
  driver = messagingv2

  [oslo_messaging_rabbit]
  amqp_durable_queues = true
  rabbit_ha_queues = true
  rabbit_retry_backoff = 2
  rabbit_retry_interval = 1

  [oslo_middleware]
  enable_proxy_headers_parsing = true

  [service_providers]
  service_provider = FIREWALL:Iptables:neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver:default
  service_provider = LOADBALANCERV2:Haproxy:neutron_lbaas.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default

  ##############################
  fwaas_driver.ini
  ##############################

  [fwaas]
  enabled = true
  driver = neutron_fwaas.services.firewall.drivers.linux.iptables_fwaas.IptablesFwaasDriver

  May someone please have a look.

To manage notifications about this bug go to:
https://bugs.launchpad.net/neutron/+bug/1715395/+subscriptions
Follow ups

[Bug 1715395] Re: FWaaS: Firewall creation fails in case of distributed routers (Pike)
From: OpenStack Infra, 2018-03-19