yahoo-eng-team team mailing list archive
-
yahoo-eng-team team
-
Mailing list archive
-
Message #87467
[Bug 1947687] [NEW] instance nework info cache refresh fails with foreign key constraint fails: instance_info_caches_instance_uuid_fkey
Public bug reported:
Instance fails to spawn during tempest. The logs shows a DB
inconsistency:
BReferenceError (pymysql.err.IntegrityError) (1452, 'Cannot add or update a child row: a foreign key constraint fails (`nova_cell1`.`instance_info_caches`, CONSTRAINT `instance_info_caches_instance_uuid_fkey` FOREIGN KEY (`instance_uuid`) REFERENCES `instances` (`uuid`))')
[SQL: INSERT INTO instance_info_caches (created_at, updated_at, deleted_at, deleted, network_info, instance_uuid) VALUES (%(created_at)s, %(updated_at)s, %(deleted_at)s, %(deleted)s, %(network_info)s, %(instance_uuid)s)]
[parameters: {'created_at': datetime.datetime(2021, 10, 18, 18, 3, 43, 474861), 'updated_at': None, 'deleted_at': None, 'deleted': 0, 'network_info': '[{"id": "9ccd333d-cfd8-4530-b5d4-31ae581f17d4", "address": "fa:16:3e:f3:44:82", "network": {"id": "81df5845-8bc5-4a75-9a60-ccc9aaa086b5", "bridge": " ... (627 characters truncated) ... : null, "qbg_params": null, "active": false, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]', 'instance_uuid': '2ba41b43-2041-4c78-a09f-5875e1f135d9'}]
Full stack trace:
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ['Traceback (most recent call last):\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/base.py", line
1799, in _execute_context\n self.dialect.do_execute(\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/default.py",
line 717, in do_execute\n cursor.execute(statement, parameters)\n', '
File "/usr/local/lib/python3.8/dist-packages/pymysql/cursors.py", line
148, in execute\n result = self._query(query)\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/cursors.py", line 310,
in _query\n conn.query(q)\n', ' File "/usr/local/lib/python3.8/dist-
packages/pymysql/connections.py", line 548, in query\n
self._affected_rows = self._read_query_result(unbuffered=unbuffered)\n',
' File "/usr/local/lib/python3.8/dist-packages/pymysql/connections.py",
line 775, in _read_query_result\n result.read()\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line
1156, in read\n first_packet = self.connection._read_packet()\n', '
File "/usr/local/lib/python3.8/dist-packages/pymysql/connections.py",
line 725, in _read_packet\n packet.raise_for_error()\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/protocol.py", line 221,
in raise_for_error\n err.raise_mysql_exception(self._data)\n', '
File "/usr/local/lib/python3.8/dist-packages/pymysql/err.py", line 143,
in raise_mysql_exception\n raise errorclass(errno, errval)\n',
"pymysql.err.IntegrityError: (1452, 'Cannot add or update a child row: a
foreign key constraint fails (`nova_cell1`.`instance_info_caches`,
CONSTRAINT `instance_info_caches_instance_uuid_fkey` FOREIGN KEY
(`instance_uuid`) REFERENCES `instances` (`uuid`))')\n", '\nThe above
exception was the direct cause of the following exception:\n\n',
'Traceback (most recent call last):\n', ' File
"/opt/stack/nova/nova/conductor/manager.py", line 137, in
_object_dispatch\n return getattr(target, method)(*args,
**kwargs)\n', ' File "/usr/local/lib/python3.8/dist-
packages/oslo_versionedobjects/base.py", line 226, in wrapper\n
return fn(self, *args, **kwargs)\n', ' File
"/opt/stack/nova/nova/objects/instance_info_cache.py", line 78, in
save\n rv = db.instance_info_cache_update(self._context,\n', ' File
"/opt/stack/nova/nova/db/utils.py", line 35, in wrapper\n return
f(*args, **kwargs)\n', ' File "/usr/local/lib/python3.8/dist-
packages/oslo_db/api.py", line 154, in wrapper\n ectxt.value =
e.inner_exc\n', ' File "/usr/local/lib/python3.8/dist-
packages/oslo_utils/excutils.py", line 227, in __exit__\n
self.force_reraise()\n', ' File "/usr/local/lib/python3.8/dist-
packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise
self.value\n', ' File "/usr/local/lib/python3.8/dist-
packages/oslo_db/api.py", line 142, in wrapper\n return f(*args,
**kwargs)\n', ' File "/opt/stack/nova/nova/db/main/api.py", line 190,
in wrapper\n return f(context, *args, **kwargs)\n', ' File
"/opt/stack/nova/nova/db/main/api.py", line 2447, in
instance_info_cache_update\n info_cache.save(context.session)\n', '
File "/usr/local/lib/python3.8/dist-
packages/oslo_db/sqlalchemy/models.py", line 48, in save\n
session.flush()\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/session.py", line 3339, in flush\n
self._flush(objects)\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/session.py", line 3479, in _flush\n
transaction.rollback(_capture_exception=True)\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/util/langhelpers.py",
line 70, in __exit__\n compat.raise_(\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/util/compat.py", line
207, in raise_\n raise exception\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/orm/session.py", line
3439, in _flush\n flush_context.execute()\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/orm/unitofwork.py",
line 456, in execute\n rec.execute(self)\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/orm/unitofwork.py",
line 630, in execute\n util.preloaded.orm_persistence.save_obj(\n', '
File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/persistence.py", line 242, in save_obj\n
_emit_insert_statements(\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/persistence.py", line 1219, in
_emit_insert_statements\n result = connection._execute_20(\n', '
File "/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/base.py",
line 1611, in _execute_20\n return meth(self, args_10style,
kwargs_10style, execution_options)\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/sql/elements.py",
line 323, in _execute_on_connection\n return
connection._execute_clauseelement(\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/base.py", line
1478, in _execute_clauseelement\n ret = self._execute_context(\n', '
File "/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/base.py",
line 1842, in _execute_context\n self._handle_dbapi_exception(\n', '
File "/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/base.py",
line 2021, in _handle_dbapi_exception\n util.raise_(newraise,
with_traceback=exc_info[2], from_=e)\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/util/compat.py", line
207, in raise_\n raise exception\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/base.py", line
1799, in _execute_context\n self.dialect.do_execute(\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/default.py",
line 717, in do_execute\n cursor.execute(statement, parameters)\n', '
File "/usr/local/lib/python3.8/dist-packages/pymysql/cursors.py", line
148, in execute\n result = self._query(query)\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/cursors.py", line 310,
in _query\n conn.query(q)\n', ' File "/usr/local/lib/python3.8/dist-
packages/pymysql/connections.py", line 548, in query\n
self._affected_rows = self._read_query_result(unbuffered=unbuffered)\n',
' File "/usr/local/lib/python3.8/dist-packages/pymysql/connections.py",
line 775, in _read_query_result\n result.read()\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line
1156, in read\n first_packet = self.connection._read_packet()\n', '
File "/usr/local/lib/python3.8/dist-packages/pymysql/connections.py",
line 725, in _read_packet\n packet.raise_for_error()\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/protocol.py", line 221,
in raise_for_error\n err.raise_mysql_exception(self._data)\n', '
File "/usr/local/lib/python3.8/dist-packages/pymysql/err.py", line 143,
in raise_mysql_exception\n raise errorclass(errno, errval)\n',
'oslo_db.exception.DBReferenceError: (pymysql.err.IntegrityError) (1452,
\'Cannot add or update a child row: a foreign key constraint fails
(`nova_cell1`.`instance_info_caches`, CONSTRAINT
`instance_info_caches_instance_uuid_fkey` FOREIGN KEY (`instance_uuid`)
REFERENCES `instances` (`uuid`))\')\n[SQL: INSERT INTO
instance_info_caches (created_at, updated_at, deleted_at, deleted,
network_info, instance_uuid) VALUES (%(created_at)s, %(updated_at)s,
%(deleted_at)s, %(deleted)s, %(network_info)s,
%(instance_uuid)s)]\n[parameters: {\'created_at\':
datetime.datetime(2021, 10, 18, 18, 3, 43, 474861), \'updated_at\':
None, \'deleted_at\': None, \'deleted\': 0, \'network_info\': \'[{"id":
"9ccd333d-cfd8-4530-b5d4-31ae581f17d4", "address": "fa:16:3e:f3:44:82",
"network": {"id": "81df5845-8bc5-4a75-9a60-ccc9aaa086b5", "bridge": "
... (627 characters truncated) ... : null, "qbg_params": null, "active":
false, "vnic_type": "normal", "profile": {}, "preserve_on_delete":
false, "delegate_create": true, "meta": {}}]\', \'instance_uuid\':
\'2ba41b43-2041-4c78-a09f-5875e1f135d9\'}]\n(Background on this error
at: https://sqlalche.me/e/14/gkpj)\n'].
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server Traceback (most recent
call last):
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py",
line 165, in _process_incoming
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server res =
self.dispatcher.dispatch(message)
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-
packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server return
self._do_dispatch(endpoint, method, ctxt, args)
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-
packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server result = func(ctxt,
**new_args)
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/opt/stack/nova/nova/exception_wrapper.py", line 71, in wrapped
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server
_emit_versioned_exception_notification(
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line
227, in __exit__
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server
self.force_reraise()
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line
200, in force_reraise
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server raise self.value
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/opt/stack/nova/nova/exception_wrapper.py", line 63, in wrapped
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server return f(self,
context, *args, **kw)
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/opt/stack/nova/nova/compute/manager.py", line 10513, in
external_instance_event
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server
self.network_api.get_instance_nw_info(
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/opt/stack/nova/nova/network/neutron.py", line 1857, in
get_instance_nw_info
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server
update_instance_cache_with_nw_info(self, context, instance,
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/opt/stack/nova/nova/network/neutron.py", line 129, in
update_instance_cache_with_nw_info
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server
LOG.exception('Failed storing info cache', instance=instance)
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line
227, in __exit__
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server
self.force_reraise()
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line
200, in force_reraise
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server raise self.value
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/opt/stack/nova/nova/network/neutron.py", line 125, in
update_instance_cache_with_nw_info
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server ic.save()
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-packages/oslo_versionedobjects/base.py",
line 209, in wrapper
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server updates, result =
self.indirection_api.object_action(
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/opt/stack/nova/nova/conductor/rpcapi.py", line 247, in object_action
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server return
cctxt.call(context, 'object_action', objinst=objinst,
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/client.py",
line 189, in call
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server result =
self.transport._send(
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-packages/oslo_messaging/transport.py",
line 123, in _send
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server return
self._driver.send(target, ctxt, message,
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-
packages/oslo_messaging/_drivers/amqpdriver.py", line 680, in send
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server return
self._send(target, ctxt, message, wait_for_reply, timeout,
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-
packages/oslo_messaging/_drivers/amqpdriver.py", line 672, in _send
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server raise result
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server
oslo_messaging.rpc.client.RemoteError: Remote error: DBReferenceError
(pymysql.err.IntegrityError) (1452, 'Cannot add or update a child row: a
foreign key constraint fails (`nova_cell1`.`instance_info_caches`,
CONSTRAINT `instance_info_caches_instance_uuid_fkey` FOREIGN KEY
(`instance_uuid`) REFERENCES `instances` (`uuid`))')
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server [SQL: INSERT INTO
instance_info_caches (created_at, updated_at, deleted_at, deleted,
network_info, instance_uuid) VALUES (%(created_at)s, %(updated_at)s,
%(deleted_at)s, %(deleted)s, %(network_info)s, %(instance_uuid)s)]
Oct 18 18:03:43.508510 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server [parameters:
{'created_at': datetime.datetime(2021, 10, 18, 18, 3, 43, 474861),
'updated_at': None, 'deleted_at': None, 'deleted': 0, 'network_info':
'[{"id": "9ccd333d-cfd8-4530-b5d4-31ae581f17d4", "address":
"fa:16:3e:f3:44:82", "network": {"id":
"81df5845-8bc5-4a75-9a60-ccc9aaa086b5", "bridge": " ... (627 characters
truncated) ... : null, "qbg_params": null, "active": false, "vnic_type":
"normal", "profile": {}, "preserve_on_delete": false, "delegate_create":
true, "meta": {}}]', 'instance_uuid':
'2ba41b43-2041-4c78-a09f-5875e1f135d9'}]
Oct 18 18:03:43.508510 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server (Background on this
error at: https://sqlalche.me/e/14/gkpj)
Oct 18 18:03:43.508510 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server ['Traceback (most
recent call last):\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/engine/base.py", line 1799, in _execute_context\n
self.dialect.do_execute(\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/engine/default.py", line 717, in do_execute\n
cursor.execute(statement, parameters)\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/cursors.py", line 148,
in execute\n result = self._query(query)\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/cursors.py", line 310,
in _query\n conn.query(q)\n', ' File "/usr/local/lib/python3.8/dist-
packages/pymysql/connections.py", line 548, in query\n
self._affected_rows = self._read_query_result(unbuffered=unbuffered)\n',
' File "/usr/local/lib/python3.8/dist-packages/pymysql/connections.py",
line 775, in _read_query_result\n result.read()\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line
1156, in read\n first_packet = self.connection._read_packet()\n', '
File "/usr/local/lib/python3.8/dist-packages/pymysql/connections.py",
line 725, in _read_packet\n packet.raise_for_error()\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/protocol.py", line 221,
in raise_for_error\n err.raise_mysql_exception(self._data)\n', '
File "/usr/local/lib/python3.8/dist-packages/pymysql/err.py", line 143,
in raise_mysql_exception\n raise errorclass(errno, errval)\n',
"pymysql.err.IntegrityError: (1452, 'Cannot add or update a child row: a
foreign key constraint fails (`nova_cell1`.`instance_info_caches`,
CONSTRAINT `instance_info_caches_instance_uuid_fkey` FOREIGN KEY
(`instance_uuid`) REFERENCES `instances` (`uuid`))')\n", '\nThe above
exception was the direct cause of the following exception:\n\n',
'Traceback (most recent call last):\n', ' File
"/opt/stack/nova/nova/conductor/manager.py", line 137, in
_object_dispatch\n return getattr(target, method)(*args,
**kwargs)\n', ' File "/usr/local/lib/python3.8/dist-
packages/oslo_versionedobjects/base.py", line 226, in wrapper\n
return fn(self, *args, **kwargs)\n', ' File
"/opt/stack/nova/nova/objects/instance_info_cache.py", line 78, in
save\n rv = db.instance_info_cache_update(self._context,\n', ' File
"/opt/stack/nova/nova/db/utils.py", line 35, in wrapper\n return
f(*args, **kwargs)\n', ' File "/usr/local/lib/python3.8/dist-
packages/oslo_db/api.py", line 154, in wrapper\n ectxt.value =
e.inner_exc\n', ' File "/usr/local/lib/python3.8/dist-
packages/oslo_utils/excutils.py", line 227, in __exit__\n
self.force_reraise()\n', ' File "/usr/local/lib/python3.8/dist-
packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise
self.value\n', ' File "/usr/local/lib/python3.8/dist-
packages/oslo_db/api.py", line 142, in wrapper\n return f(*args,
**kwargs)\n', ' File "/opt/stack/nova/nova/db/main/api.py", line 190,
in wrapper\n return f(context, *args, **kwargs)\n', ' File
"/opt/stack/nova/nova/db/main/api.py", line 2447, in
instance_info_cache_update\n info_cache.save(context.session)\n', '
File "/usr/local/lib/python3.8/dist-
packages/oslo_db/sqlalchemy/models.py", line 48, in save\n
session.flush()\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/session.py", line 3339, in flush\n
self._flush(objects)\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/session.py", line 3479, in _flush\n
transaction.rollback(_capture_exception=True)\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/util/langhelpers.py",
line 70, in __exit__\n compat.raise_(\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/util/compat.py", line
207, in raise_\n raise exception\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/orm/session.py", line
3439, in _flush\n flush_context.execute()\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/orm/unitofwork.py",
line 456, in execute\n rec.execute(self)\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/orm/unitofwork.py",
line 630, in execute\n util.preloaded.orm_persistence.save_obj(\n', '
File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/persistence.py", line 242, in save_obj\n
_emit_insert_statements(\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/persistence.py", line 1219, in
_emit_insert_statements\n result = connection._execute_20(\n', '
File "/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/base.py",
line 1611, in _execute_20\n return meth(self, args_10style,
kwargs_10style, execution_options)\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/sql/elements.py",
line 323, in _execute_on_connection\n return
connection._execute_clauseelement(\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/base.py", line
1478, in _execute_clauseelement\n ret = self._execute_context(\n', '
File "/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/base.py",
line 1842, in _execute_context\n self._handle_dbapi_exception(\n', '
File "/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/base.py",
line 2021, in _handle_dbapi_exception\n util.raise_(newraise,
with_traceback=exc_info[2], from_=e)\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/util/compat.py", line
207, in raise_\n raise exception\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/base.py", line
1799, in _execute_context\n self.dialect.do_execute(\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/default.py",
line 717, in do_execute\n cursor.execute(statement, parameters)\n', '
File "/usr/local/lib/python3.8/dist-packages/pymysql/cursors.py", line
148, in execute\n result = self._query(query)\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/cursors.py", line 310,
in _query\n conn.query(q)\n', ' File "/usr/local/lib/python3.8/dist-
packages/pymysql/connections.py", line 548, in query\n
self._affected_rows = self._read_query_result(unbuffered=unbuffered)\n',
' File "/usr/local/lib/python3.8/dist-packages/pymysql/connections.py",
line 775, in _read_query_result\n result.read()\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line
1156, in read\n first_packet = self.connection._read_packet()\n', '
File "/usr/local/lib/python3.8/dist-packages/pymysql/connections.py",
line 725, in _read_packet\n packet.raise_for_error()\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/protocol.py", line 221,
in raise_for_error\n err.raise_mysql_exception(self._data)\n', '
File "/usr/local/lib/python3.8/dist-packages/pymysql/err.py", line 143,
in raise_mysql_exception\n raise errorclass(errno, errval)\n',
'oslo_db.exception.DBReferenceError: (pymysql.err.IntegrityError) (1452,
\'Cannot add or update a child row: a foreign key constraint fails
(`nova_cell1`.`instance_info_caches`, CONSTRAINT
`instance_info_caches_instance_uuid_fkey` FOREIGN KEY (`instance_uuid`)
REFERENCES `instances` (`uuid`))\')\n[SQL: INSERT INTO
instance_info_caches (created_at, updated_at, deleted_at, deleted,
network_info, instance_uuid) VALUES (%(created_at)s, %(updated_at)s,
%(deleted_at)s, %(deleted)s, %(network_info)s,
%(instance_uuid)s)]\n[parameters: {\'created_at\':
datetime.datetime(2021, 10, 18, 18, 3, 43, 474861), \'updated_at\':
None, \'deleted_at\': None, \'deleted\': 0, \'network_info\': \'[{"id":
"9ccd333d-cfd8-4530-b5d4-31ae581f17d4", "address": "fa:16:3e:f3:44:82",
"network": {"id": "81df5845-8bc5-4a75-9a60-ccc9aaa086b5", "bridge": "
... (627 characters truncated) ... : null, "qbg_params": null, "active":
false, "vnic_type": "normal", "profile": {}, "preserve_on_delete":
false, "delegate_create": true, "meta": {}}]\', \'instance_uuid\':
\'2ba41b43-2041-4c78-a09f-5875e1f135d9\'}]\n(Background on this error
at: https://sqlalche.me/e/14/gkpj)\n'].
Oct 18 18:03:43.508510 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server
Example failure:
https://zuul.opendev.org/t/openstack/build/f90600bfb6f54b9bbf3ec2c45fff1c1d/log/controller/logs/screen-n-cpu.txt#10377
** Affects: nova
Importance: Critical
Status: New
** Tags: gate-failure
** Tags added: gate-failure
** Summary changed:
- instance nework infor refresh fails with foreign key constraint fails: instance_info_caches_instance_uuid_fkey
+ instance nework info cache refresh fails with foreign key constraint fails: instance_info_caches_instance_uuid_fkey
--
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to OpenStack Compute (nova).
https://bugs.launchpad.net/bugs/1947687
Title:
instance nework info cache refresh fails with foreign key constraint
fails: instance_info_caches_instance_uuid_fkey
Status in OpenStack Compute (nova):
New
Bug description:
Instance fails to spawn during tempest. The logs shows a DB
inconsistency:
BReferenceError (pymysql.err.IntegrityError) (1452, 'Cannot add or update a child row: a foreign key constraint fails (`nova_cell1`.`instance_info_caches`, CONSTRAINT `instance_info_caches_instance_uuid_fkey` FOREIGN KEY (`instance_uuid`) REFERENCES `instances` (`uuid`))')
[SQL: INSERT INTO instance_info_caches (created_at, updated_at, deleted_at, deleted, network_info, instance_uuid) VALUES (%(created_at)s, %(updated_at)s, %(deleted_at)s, %(deleted)s, %(network_info)s, %(instance_uuid)s)]
[parameters: {'created_at': datetime.datetime(2021, 10, 18, 18, 3, 43, 474861), 'updated_at': None, 'deleted_at': None, 'deleted': 0, 'network_info': '[{"id": "9ccd333d-cfd8-4530-b5d4-31ae581f17d4", "address": "fa:16:3e:f3:44:82", "network": {"id": "81df5845-8bc5-4a75-9a60-ccc9aaa086b5", "bridge": " ... (627 characters truncated) ... : null, "qbg_params": null, "active": false, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]', 'instance_uuid': '2ba41b43-2041-4c78-a09f-5875e1f135d9'}]
Full stack trace:
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ['Traceback (most recent call last):\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/base.py",
line 1799, in _execute_context\n self.dialect.do_execute(\n', '
File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/engine/default.py", line 717, in do_execute\n
cursor.execute(statement, parameters)\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/cursors.py", line 148,
in execute\n result = self._query(query)\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/cursors.py", line 310,
in _query\n conn.query(q)\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line
548, in query\n self._affected_rows =
self._read_query_result(unbuffered=unbuffered)\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line
775, in _read_query_result\n result.read()\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line
1156, in read\n first_packet = self.connection._read_packet()\n', '
File "/usr/local/lib/python3.8/dist-packages/pymysql/connections.py",
line 725, in _read_packet\n packet.raise_for_error()\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/protocol.py", line
221, in raise_for_error\n err.raise_mysql_exception(self._data)\n',
' File "/usr/local/lib/python3.8/dist-packages/pymysql/err.py", line
143, in raise_mysql_exception\n raise errorclass(errno, errval)\n',
"pymysql.err.IntegrityError: (1452, 'Cannot add or update a child row:
a foreign key constraint fails (`nova_cell1`.`instance_info_caches`,
CONSTRAINT `instance_info_caches_instance_uuid_fkey` FOREIGN KEY
(`instance_uuid`) REFERENCES `instances` (`uuid`))')\n", '\nThe above
exception was the direct cause of the following exception:\n\n',
'Traceback (most recent call last):\n', ' File
"/opt/stack/nova/nova/conductor/manager.py", line 137, in
_object_dispatch\n return getattr(target, method)(*args,
**kwargs)\n', ' File "/usr/local/lib/python3.8/dist-
packages/oslo_versionedobjects/base.py", line 226, in wrapper\n
return fn(self, *args, **kwargs)\n', ' File
"/opt/stack/nova/nova/objects/instance_info_cache.py", line 78, in
save\n rv = db.instance_info_cache_update(self._context,\n', '
File "/opt/stack/nova/nova/db/utils.py", line 35, in wrapper\n
return f(*args, **kwargs)\n', ' File "/usr/local/lib/python3.8/dist-
packages/oslo_db/api.py", line 154, in wrapper\n ectxt.value =
e.inner_exc\n', ' File "/usr/local/lib/python3.8/dist-
packages/oslo_utils/excutils.py", line 227, in __exit__\n
self.force_reraise()\n', ' File "/usr/local/lib/python3.8/dist-
packages/oslo_utils/excutils.py", line 200, in force_reraise\n
raise self.value\n', ' File "/usr/local/lib/python3.8/dist-
packages/oslo_db/api.py", line 142, in wrapper\n return f(*args,
**kwargs)\n', ' File "/opt/stack/nova/nova/db/main/api.py", line 190,
in wrapper\n return f(context, *args, **kwargs)\n', ' File
"/opt/stack/nova/nova/db/main/api.py", line 2447, in
instance_info_cache_update\n info_cache.save(context.session)\n', '
File "/usr/local/lib/python3.8/dist-
packages/oslo_db/sqlalchemy/models.py", line 48, in save\n
session.flush()\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/session.py", line 3339, in flush\n
self._flush(objects)\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/session.py", line 3479, in _flush\n
transaction.rollback(_capture_exception=True)\n', ' File
"/usr/local/lib/python3.8/dist-
packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__\n
compat.raise_(\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/util/compat.py", line 207, in raise_\n raise
exception\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/session.py", line 3439, in _flush\n
flush_context.execute()\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/unitofwork.py", line 456, in execute\n
rec.execute(self)\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/unitofwork.py", line 630, in execute\n
util.preloaded.orm_persistence.save_obj(\n', ' File
"/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/persistence.py", line 242, in save_obj\n
_emit_insert_statements(\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/persistence.py", line 1219, in
_emit_insert_statements\n result = connection._execute_20(\n', '
File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/engine/base.py", line 1611, in _execute_20\n
return meth(self, args_10style, kwargs_10style, execution_options)\n',
' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/sql/elements.py", line 323, in
_execute_on_connection\n return
connection._execute_clauseelement(\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/base.py",
line 1478, in _execute_clauseelement\n ret =
self._execute_context(\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/engine/base.py", line 1842, in _execute_context\n
self._handle_dbapi_exception(\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/base.py",
line 2021, in _handle_dbapi_exception\n util.raise_(newraise,
with_traceback=exc_info[2], from_=e)\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/util/compat.py",
line 207, in raise_\n raise exception\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/base.py",
line 1799, in _execute_context\n self.dialect.do_execute(\n', '
File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/engine/default.py", line 717, in do_execute\n
cursor.execute(statement, parameters)\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/cursors.py", line 148,
in execute\n result = self._query(query)\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/cursors.py", line 310,
in _query\n conn.query(q)\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line
548, in query\n self._affected_rows =
self._read_query_result(unbuffered=unbuffered)\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line
775, in _read_query_result\n result.read()\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line
1156, in read\n first_packet = self.connection._read_packet()\n', '
File "/usr/local/lib/python3.8/dist-packages/pymysql/connections.py",
line 725, in _read_packet\n packet.raise_for_error()\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/protocol.py", line
221, in raise_for_error\n err.raise_mysql_exception(self._data)\n',
' File "/usr/local/lib/python3.8/dist-packages/pymysql/err.py", line
143, in raise_mysql_exception\n raise errorclass(errno, errval)\n',
'oslo_db.exception.DBReferenceError: (pymysql.err.IntegrityError)
(1452, \'Cannot add or update a child row: a foreign key constraint
fails (`nova_cell1`.`instance_info_caches`, CONSTRAINT
`instance_info_caches_instance_uuid_fkey` FOREIGN KEY
(`instance_uuid`) REFERENCES `instances` (`uuid`))\')\n[SQL: INSERT
INTO instance_info_caches (created_at, updated_at, deleted_at,
deleted, network_info, instance_uuid) VALUES (%(created_at)s,
%(updated_at)s, %(deleted_at)s, %(deleted)s, %(network_info)s,
%(instance_uuid)s)]\n[parameters: {\'created_at\':
datetime.datetime(2021, 10, 18, 18, 3, 43, 474861), \'updated_at\':
None, \'deleted_at\': None, \'deleted\': 0, \'network_info\':
\'[{"id": "9ccd333d-cfd8-4530-b5d4-31ae581f17d4", "address":
"fa:16:3e:f3:44:82", "network": {"id":
"81df5845-8bc5-4a75-9a60-ccc9aaa086b5", "bridge": " ... (627
characters truncated) ... : null, "qbg_params": null, "active": false,
"vnic_type": "normal", "profile": {}, "preserve_on_delete": false,
"delegate_create": true, "meta": {}}]\', \'instance_uuid\':
\'2ba41b43-2041-4c78-a09f-5875e1f135d9\'}]\n(Background on this error
at: https://sqlalche.me/e/14/gkpj)\n'].
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server Traceback (most
recent call last):
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/server.py",
line 165, in _process_incoming
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server res =
self.dispatcher.dispatch(message)
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-
packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server return
self._do_dispatch(endpoint, method, ctxt, args)
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-
packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server result =
func(ctxt, **new_args)
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/opt/stack/nova/nova/exception_wrapper.py", line 71, in wrapped
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server
_emit_versioned_exception_notification(
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line
227, in __exit__
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server
self.force_reraise()
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line
200, in force_reraise
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server raise self.value
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/opt/stack/nova/nova/exception_wrapper.py", line 63, in wrapped
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server return f(self,
context, *args, **kw)
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/opt/stack/nova/nova/compute/manager.py", line 10513, in
external_instance_event
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server
self.network_api.get_instance_nw_info(
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/opt/stack/nova/nova/network/neutron.py", line 1857, in
get_instance_nw_info
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server
update_instance_cache_with_nw_info(self, context, instance,
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/opt/stack/nova/nova/network/neutron.py", line 129, in
update_instance_cache_with_nw_info
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server
LOG.exception('Failed storing info cache', instance=instance)
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line
227, in __exit__
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server
self.force_reraise()
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-packages/oslo_utils/excutils.py", line
200, in force_reraise
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server raise self.value
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/opt/stack/nova/nova/network/neutron.py", line 125, in
update_instance_cache_with_nw_info
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server ic.save()
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-
packages/oslo_versionedobjects/base.py", line 209, in wrapper
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server updates, result =
self.indirection_api.object_action(
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/opt/stack/nova/nova/conductor/rpcapi.py", line 247, in object_action
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server return
cctxt.call(context, 'object_action', objinst=objinst,
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-packages/oslo_messaging/rpc/client.py",
line 189, in call
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server result =
self.transport._send(
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-packages/oslo_messaging/transport.py",
line 123, in _send
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server return
self._driver.send(target, ctxt, message,
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-
packages/oslo_messaging/_drivers/amqpdriver.py", line 680, in send
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server return
self._send(target, ctxt, message, wait_for_reply, timeout,
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server File
"/usr/local/lib/python3.8/dist-
packages/oslo_messaging/_drivers/amqpdriver.py", line 672, in _send
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server raise result
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server
oslo_messaging.rpc.client.RemoteError: Remote error: DBReferenceError
(pymysql.err.IntegrityError) (1452, 'Cannot add or update a child row:
a foreign key constraint fails (`nova_cell1`.`instance_info_caches`,
CONSTRAINT `instance_info_caches_instance_uuid_fkey` FOREIGN KEY
(`instance_uuid`) REFERENCES `instances` (`uuid`))')
Oct 18 18:03:43.504910 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server [SQL: INSERT INTO
instance_info_caches (created_at, updated_at, deleted_at, deleted,
network_info, instance_uuid) VALUES (%(created_at)s, %(updated_at)s,
%(deleted_at)s, %(deleted)s, %(network_info)s, %(instance_uuid)s)]
Oct 18 18:03:43.508510 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server [parameters:
{'created_at': datetime.datetime(2021, 10, 18, 18, 3, 43, 474861),
'updated_at': None, 'deleted_at': None, 'deleted': 0, 'network_info':
'[{"id": "9ccd333d-cfd8-4530-b5d4-31ae581f17d4", "address":
"fa:16:3e:f3:44:82", "network": {"id":
"81df5845-8bc5-4a75-9a60-ccc9aaa086b5", "bridge": " ... (627
characters truncated) ... : null, "qbg_params": null, "active": false,
"vnic_type": "normal", "profile": {}, "preserve_on_delete": false,
"delegate_create": true, "meta": {}}]', 'instance_uuid':
'2ba41b43-2041-4c78-a09f-5875e1f135d9'}]
Oct 18 18:03:43.508510 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server (Background on this
error at: https://sqlalche.me/e/14/gkpj)
Oct 18 18:03:43.508510 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server ['Traceback (most
recent call last):\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/engine/base.py", line 1799, in _execute_context\n
self.dialect.do_execute(\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/engine/default.py", line 717, in do_execute\n
cursor.execute(statement, parameters)\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/cursors.py", line 148,
in execute\n result = self._query(query)\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/cursors.py", line 310,
in _query\n conn.query(q)\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line
548, in query\n self._affected_rows =
self._read_query_result(unbuffered=unbuffered)\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line
775, in _read_query_result\n result.read()\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line
1156, in read\n first_packet = self.connection._read_packet()\n', '
File "/usr/local/lib/python3.8/dist-packages/pymysql/connections.py",
line 725, in _read_packet\n packet.raise_for_error()\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/protocol.py", line
221, in raise_for_error\n err.raise_mysql_exception(self._data)\n',
' File "/usr/local/lib/python3.8/dist-packages/pymysql/err.py", line
143, in raise_mysql_exception\n raise errorclass(errno, errval)\n',
"pymysql.err.IntegrityError: (1452, 'Cannot add or update a child row:
a foreign key constraint fails (`nova_cell1`.`instance_info_caches`,
CONSTRAINT `instance_info_caches_instance_uuid_fkey` FOREIGN KEY
(`instance_uuid`) REFERENCES `instances` (`uuid`))')\n", '\nThe above
exception was the direct cause of the following exception:\n\n',
'Traceback (most recent call last):\n', ' File
"/opt/stack/nova/nova/conductor/manager.py", line 137, in
_object_dispatch\n return getattr(target, method)(*args,
**kwargs)\n', ' File "/usr/local/lib/python3.8/dist-
packages/oslo_versionedobjects/base.py", line 226, in wrapper\n
return fn(self, *args, **kwargs)\n', ' File
"/opt/stack/nova/nova/objects/instance_info_cache.py", line 78, in
save\n rv = db.instance_info_cache_update(self._context,\n', '
File "/opt/stack/nova/nova/db/utils.py", line 35, in wrapper\n
return f(*args, **kwargs)\n', ' File "/usr/local/lib/python3.8/dist-
packages/oslo_db/api.py", line 154, in wrapper\n ectxt.value =
e.inner_exc\n', ' File "/usr/local/lib/python3.8/dist-
packages/oslo_utils/excutils.py", line 227, in __exit__\n
self.force_reraise()\n', ' File "/usr/local/lib/python3.8/dist-
packages/oslo_utils/excutils.py", line 200, in force_reraise\n
raise self.value\n', ' File "/usr/local/lib/python3.8/dist-
packages/oslo_db/api.py", line 142, in wrapper\n return f(*args,
**kwargs)\n', ' File "/opt/stack/nova/nova/db/main/api.py", line 190,
in wrapper\n return f(context, *args, **kwargs)\n', ' File
"/opt/stack/nova/nova/db/main/api.py", line 2447, in
instance_info_cache_update\n info_cache.save(context.session)\n', '
File "/usr/local/lib/python3.8/dist-
packages/oslo_db/sqlalchemy/models.py", line 48, in save\n
session.flush()\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/session.py", line 3339, in flush\n
self._flush(objects)\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/session.py", line 3479, in _flush\n
transaction.rollback(_capture_exception=True)\n', ' File
"/usr/local/lib/python3.8/dist-
packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__\n
compat.raise_(\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/util/compat.py", line 207, in raise_\n raise
exception\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/session.py", line 3439, in _flush\n
flush_context.execute()\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/unitofwork.py", line 456, in execute\n
rec.execute(self)\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/unitofwork.py", line 630, in execute\n
util.preloaded.orm_persistence.save_obj(\n', ' File
"/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/persistence.py", line 242, in save_obj\n
_emit_insert_statements(\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/orm/persistence.py", line 1219, in
_emit_insert_statements\n result = connection._execute_20(\n', '
File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/engine/base.py", line 1611, in _execute_20\n
return meth(self, args_10style, kwargs_10style, execution_options)\n',
' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/sql/elements.py", line 323, in
_execute_on_connection\n return
connection._execute_clauseelement(\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/base.py",
line 1478, in _execute_clauseelement\n ret =
self._execute_context(\n', ' File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/engine/base.py", line 1842, in _execute_context\n
self._handle_dbapi_exception(\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/base.py",
line 2021, in _handle_dbapi_exception\n util.raise_(newraise,
with_traceback=exc_info[2], from_=e)\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/util/compat.py",
line 207, in raise_\n raise exception\n', ' File
"/usr/local/lib/python3.8/dist-packages/sqlalchemy/engine/base.py",
line 1799, in _execute_context\n self.dialect.do_execute(\n', '
File "/usr/local/lib/python3.8/dist-
packages/sqlalchemy/engine/default.py", line 717, in do_execute\n
cursor.execute(statement, parameters)\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/cursors.py", line 148,
in execute\n result = self._query(query)\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/cursors.py", line 310,
in _query\n conn.query(q)\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line
548, in query\n self._affected_rows =
self._read_query_result(unbuffered=unbuffered)\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line
775, in _read_query_result\n result.read()\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line
1156, in read\n first_packet = self.connection._read_packet()\n', '
File "/usr/local/lib/python3.8/dist-packages/pymysql/connections.py",
line 725, in _read_packet\n packet.raise_for_error()\n', ' File
"/usr/local/lib/python3.8/dist-packages/pymysql/protocol.py", line
221, in raise_for_error\n err.raise_mysql_exception(self._data)\n',
' File "/usr/local/lib/python3.8/dist-packages/pymysql/err.py", line
143, in raise_mysql_exception\n raise errorclass(errno, errval)\n',
'oslo_db.exception.DBReferenceError: (pymysql.err.IntegrityError)
(1452, \'Cannot add or update a child row: a foreign key constraint
fails (`nova_cell1`.`instance_info_caches`, CONSTRAINT
`instance_info_caches_instance_uuid_fkey` FOREIGN KEY
(`instance_uuid`) REFERENCES `instances` (`uuid`))\')\n[SQL: INSERT
INTO instance_info_caches (created_at, updated_at, deleted_at,
deleted, network_info, instance_uuid) VALUES (%(created_at)s,
%(updated_at)s, %(deleted_at)s, %(deleted)s, %(network_info)s,
%(instance_uuid)s)]\n[parameters: {\'created_at\':
datetime.datetime(2021, 10, 18, 18, 3, 43, 474861), \'updated_at\':
None, \'deleted_at\': None, \'deleted\': 0, \'network_info\':
\'[{"id": "9ccd333d-cfd8-4530-b5d4-31ae581f17d4", "address":
"fa:16:3e:f3:44:82", "network": {"id":
"81df5845-8bc5-4a75-9a60-ccc9aaa086b5", "bridge": " ... (627
characters truncated) ... : null, "qbg_params": null, "active": false,
"vnic_type": "normal", "profile": {}, "preserve_on_delete": false,
"delegate_create": true, "meta": {}}]\', \'instance_uuid\':
\'2ba41b43-2041-4c78-a09f-5875e1f135d9\'}]\n(Background on this error
at: https://sqlalche.me/e/14/gkpj)\n'].
Oct 18 18:03:43.508510 ubuntu-focal-ovh-gra1-0026989505 nova-
compute[110606]: ERROR oslo_messaging.rpc.server
Example failure:
https://zuul.opendev.org/t/openstack/build/f90600bfb6f54b9bbf3ec2c45fff1c1d/log/controller/logs/screen-n-cpu.txt#10377
To manage notifications about this bug go to:
https://bugs.launchpad.net/nova/+bug/1947687/+subscriptions
