yahoo-eng-team team mailing list archive

Thread
Date
[Bug 2020195] [NEW] [ovn-octavia-provider] functional test intermittently fail with DB error: Cursor needed to be reset because of commit/rollback and can no longer be fetched from

To: yahoo-eng-team@xxxxxxxxxxxxxxxxxxx
From: Fernando Royo <2020195@xxxxxxxxxxxxxxxxxx>
Date: Fri, 19 May 2023 12:42:10 -0000
Reply-to: Bug 2020195 <2020195@xxxxxxxxxxxxxxxxxx>
Sender: noreply@xxxxxxxxxxxxx
Public bug reported:

Trace:

Captured traceback:
~~~~~~~~~~~~~~~~~~~
    Traceback (most recent call last):

      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/cursor.py", line 977, in fetchall
    rows = dbapi_cursor.fetchall()

    sqlite3.InterfaceError: Cursor needed to be reset because of
commit/rollback and can no longer be fetched from.

    
The above exception was the direct cause of the following exception:


    Traceback (most recent call last):

      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron/tests/base.py", line 178, in func
    return f(self, *args, **kwargs)
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/ovn_octavia_provider/tests/functional/test_driver.py", line 43, in test_create_lb_custom_network_retry
    self._create_load_balancer_custom_lr_ls_and_validate(
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/ovn_octavia_provider/tests/functional/base.py", line 360, in _create_load_balancer_custom_lr_ls_and_validate
    net_info.append(self._create_net("n2", "10.0.2.0/24", r_id))
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/ovn_octavia_provider/tests/functional/base.py", line 282, in _create_net
    self.l3_plugin.add_router_interface(
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron/services/ovn_l3/plugin.py", line 250, in add_router_interface
    with excutils.save_and_reraise_exception():
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__
    self.force_reraise()
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise
    raise self.value
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron/services/ovn_l3/plugin.py", line 247, in add_router_interface
    self._ovn_client.create_router_port(
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron/plugins/ml2/drivers/ovn/mech_driver/ovsdb/ovn_client.py", line 1691, in create_router_port
    router = self._l3_plugin.get_router(context, router_id)
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron_lib/db/api.py", line 223, in wrapped
    return f_with_retry(*args, **kwargs,
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron_lib/db/api.py", line 137, in wrapped
    with excutils.save_and_reraise_exception():
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__
    self.force_reraise()
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise
    raise self.value
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron_lib/db/api.py", line 135, in wrapped
    return f(*args, **kwargs)
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_db/api.py", line 144, in wrapper
    with excutils.save_and_reraise_exception() as ectxt:
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__
    self.force_reraise()
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise
    raise self.value
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_db/api.py", line 142, in wrapper
    return f(*args, **kwargs)
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron_lib/db/api.py", line 183, in wrapped
    with excutils.save_and_reraise_exception():
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__
    self.force_reraise()
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise
    raise self.value
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron_lib/db/api.py", line 181, in wrapped
    return f(*dup_args, **dup_kwargs)
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron/db/l3_db.py", line 646, in get_router
    router = self._get_router(context, id)
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_db/sqlalchemy/enginefacade.py", line 1022, in wrapper
    return fn(*args, **kwargs)
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron/db/l3_db.py", line 205, in _get_router
    router = model_query.get_by_id(
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron_lib/db/model_query.py", line 169, in get_by_id
    return query.filter(model.id == object_id).one()
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/orm/query.py", line 2869, in one
    return self._iter().one()
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/result.py", line 1476, in one
    return self._only_one_row(
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/result.py", line 559, in _only_one_row
    row = onerow(hard_close=True)
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/result.py", line 1340, in _fetchone_impl
    return self._real_result._fetchone_impl(hard_close=hard_close)
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/result.py", line 1743, in _fetchone_impl
    row = next(self.iterator, _NO_ROW)
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/orm/loading.py", line 147, in chunks
    fetch = cursor._raw_all_rows()
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/result.py", line 392, in _raw_all_rows
    rows = self._fetchall_impl()
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/cursor.py", line 1805, in _fetchall_impl
    return self.cursor_strategy.fetchall(self, self.cursor)
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/cursor.py", line 981, in fetchall
    self.handle_exception(result, dbapi_cursor, e)
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/cursor.py", line 941, in handle_exception
    result.connection._handle_dbapi_exception(
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 2122, in _handle_dbapi_exception
    util.raise_(newraise, with_traceback=exc_info[2], from_=e)
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/util/compat.py", line 208, in raise_
    raise exception
      File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/cursor.py", line 977, in fetchall
    rows = dbapi_cursor.fetchall()

    oslo_db.exception.DBError: (sqlite3.InterfaceError) Cursor needed to be reset because of commit/rollback and can no longer be fetched from.
(Background on this error at: https://sqlalche.me/e/14/rvf5)

It happends randomnly on functional tests, not related to any specific
one.

** Affects: neutron
     Importance: Undecided
     Assignee: Fernando Royo (froyoredhat)
         Status: New


** Tags: ovn-octavia-provider

** Changed in: neutron
     Assignee: (unassigned) => Fernando Royo (froyoredhat)

** Summary changed:

-  functional test intermittently fail with DB error: Cursor needed to be reset because of commit/rollback and can no longer be fetched from
+ [ovn-octavia-provider] functional test intermittently fail with DB error: Cursor needed to be reset because of commit/rollback and can no longer be fetched from

-- 
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to neutron.
https://bugs.launchpad.net/bugs/2020195

Title:
  [ovn-octavia-provider] functional test intermittently fail with DB
  error: Cursor needed to be reset because of commit/rollback and can no
  longer be fetched from

Status in neutron:
  New

Bug description:
  Trace:

  Captured traceback:
  ~~~~~~~~~~~~~~~~~~~
      Traceback (most recent call last):

        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/cursor.py", line 977, in fetchall
      rows = dbapi_cursor.fetchall()

      sqlite3.InterfaceError: Cursor needed to be reset because of
  commit/rollback and can no longer be fetched from.

      
  The above exception was the direct cause of the following exception:

  
      Traceback (most recent call last):

        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron/tests/base.py", line 178, in func
      return f(self, *args, **kwargs)
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/ovn_octavia_provider/tests/functional/test_driver.py", line 43, in test_create_lb_custom_network_retry
      self._create_load_balancer_custom_lr_ls_and_validate(
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/ovn_octavia_provider/tests/functional/base.py", line 360, in _create_load_balancer_custom_lr_ls_and_validate
      net_info.append(self._create_net("n2", "10.0.2.0/24", r_id))
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/ovn_octavia_provider/tests/functional/base.py", line 282, in _create_net
      self.l3_plugin.add_router_interface(
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron/services/ovn_l3/plugin.py", line 250, in add_router_interface
      with excutils.save_and_reraise_exception():
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__
      self.force_reraise()
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise
      raise self.value
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron/services/ovn_l3/plugin.py", line 247, in add_router_interface
      self._ovn_client.create_router_port(
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron/plugins/ml2/drivers/ovn/mech_driver/ovsdb/ovn_client.py", line 1691, in create_router_port
      router = self._l3_plugin.get_router(context, router_id)
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron_lib/db/api.py", line 223, in wrapped
      return f_with_retry(*args, **kwargs,
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron_lib/db/api.py", line 137, in wrapped
      with excutils.save_and_reraise_exception():
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__
      self.force_reraise()
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise
      raise self.value
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron_lib/db/api.py", line 135, in wrapped
      return f(*args, **kwargs)
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_db/api.py", line 144, in wrapper
      with excutils.save_and_reraise_exception() as ectxt:
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__
      self.force_reraise()
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise
      raise self.value
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_db/api.py", line 142, in wrapper
      return f(*args, **kwargs)
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron_lib/db/api.py", line 183, in wrapped
      with excutils.save_and_reraise_exception():
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__
      self.force_reraise()
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise
      raise self.value
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron_lib/db/api.py", line 181, in wrapped
      return f(*dup_args, **dup_kwargs)
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron/db/l3_db.py", line 646, in get_router
      router = self._get_router(context, id)
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/oslo_db/sqlalchemy/enginefacade.py", line 1022, in wrapper
      return fn(*args, **kwargs)
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron/db/l3_db.py", line 205, in _get_router
      router = model_query.get_by_id(
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/neutron_lib/db/model_query.py", line 169, in get_by_id
      return query.filter(model.id == object_id).one()
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/orm/query.py", line 2869, in one
      return self._iter().one()
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/result.py", line 1476, in one
      return self._only_one_row(
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/result.py", line 559, in _only_one_row
      row = onerow(hard_close=True)
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/result.py", line 1340, in _fetchone_impl
      return self._real_result._fetchone_impl(hard_close=hard_close)
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/result.py", line 1743, in _fetchone_impl
      row = next(self.iterator, _NO_ROW)
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/orm/loading.py", line 147, in chunks
      fetch = cursor._raw_all_rows()
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/result.py", line 392, in _raw_all_rows
      rows = self._fetchall_impl()
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/cursor.py", line 1805, in _fetchall_impl
      return self.cursor_strategy.fetchall(self, self.cursor)
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/cursor.py", line 981, in fetchall
      self.handle_exception(result, dbapi_cursor, e)
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/cursor.py", line 941, in handle_exception
      result.connection._handle_dbapi_exception(
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/base.py", line 2122, in _handle_dbapi_exception
      util.raise_(newraise, with_traceback=exc_info[2], from_=e)
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/util/compat.py", line 208, in raise_
      raise exception
        File "/home/zuul/src/opendev.org/openstack/ovn-octavia-provider/.tox/dsvm-functional/lib/python3.10/site-packages/sqlalchemy/engine/cursor.py", line 977, in fetchall
      rows = dbapi_cursor.fetchall()

      oslo_db.exception.DBError: (sqlite3.InterfaceError) Cursor needed to be reset because of commit/rollback and can no longer be fetched from.
  (Background on this error at: https://sqlalche.me/e/14/rvf5)

  It happends randomnly on functional tests, not related to any specific
  one.

To manage notifications about this bug go to:
https://bugs.launchpad.net/neutron/+bug/2020195/+subscriptions