yahoo-eng-team team mailing list archive
-
yahoo-eng-team team
-
Mailing list archive
-
Message #79783
[Bug 1841907] [NEW] Neutron bootstrap failing on Ubuntu bionic with Cannot change column 'network_id
Public bug reported:
Neutron bootstrap is currently failing on Ubuntu bionic (kolla-ansible-
ubuntu-source jobs) with the following error:
INFO [alembic.runtime.migration] Running upgrade 63fd95af7dcd -> c613d0b82681
Traceback (most recent call last):
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1244, in _execute_context
cursor, statement, parameters, context
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 552, in do_execute
cursor.execute(statement, parameters)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/cursors.py", line 170, in execute
result = self._query(query)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/cursors.py", line 328, in _query
conn.query(q)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/connections.py", line 517, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/connections.py", line 732, in _read_query_result
result.read()
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/connections.py", line 1075, in read
first_packet = self.connection._read_packet()
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/connections.py", line 684, in _read_packet
packet.check_error()
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/protocol.py", line 220, in check_error
err.raise_mysql_exception(self._data)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/err.py", line 109, in raise_mysql_exception
raise errorclass(errno, errval)
pymysql.err.InternalError: (1832, "Cannot change column 'network_id': used in a foreign key constraint 'subnets_ibfk_1'")
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/var/lib/kolla/venv/bin/neutron-db-manage", line 10, in <module>
sys.exit(main())
File "/var/lib/kolla/venv/lib/python3.6/site-packages/neutron/db/migration/cli.py", line 657, in main
return_val |= bool(CONF.command.func(config, CONF.command.name))
File "/var/lib/kolla/venv/lib/python3.6/site-packages/neutron/db/migration/cli.py", line 181, in do_upgrade
desc=branch, sql=CONF.command.sql)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/neutron/db/migration/cli.py", line 82, in do_alembic_command
getattr(alembic_command, cmd)(config, *args, **kwargs)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/command.py", line 276, in upgrade
script.run_env()
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/script/base.py", line 475, in run_env
util.load_python_file(self.dir, "env.py")
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/util/pyfiles.py", line 90, in load_python_file
module = load_module_py(module_id, path)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/util/compat.py", line 177, in load_module_py
spec.loader.exec_module(module)
File "<frozen importlib._bootstrap_external>", line 678, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/var/lib/kolla/venv/lib/python3.6/site-packages/neutron/db/migration/alembic_migrations/env.py", line 120, in <module>
run_migrations_online()
File "/var/lib/kolla/venv/lib/python3.6/site-packages/neutron/db/migration/alembic_migrations/env.py", line 114, in run_migrations_online
context.run_migrations()
File "<string>", line 8, in run_migrations
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/runtime/environment.py", line 839, in run_migrations
self.get_context().run_migrations(**kw)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/runtime/migration.py", line 362, in run_migrations
step.migration_fn(**kw)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/neutron/db/migration/alembic_migrations/versions/train/expand/c613d0b82681_subnet_force_network_id.py", line 35, in upgrade
existing_type=sa.String(36))
File "<string>", line 8, in alter_column
File "<string>", line 3, in alter_column
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/operations/ops.py", line 1775, in alter_column
return operations.invoke(alt)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/operations/base.py", line 345, in invoke
return fn(self, operation)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/operations/toimpl.py", line 56, in alter_column
**operation.kw
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/ddl/mysql.py", line 96, in alter_column
else existing_comment,
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/ddl/impl.py", line 134, in _exec
return conn.execute(construct, *multiparams, **params)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 988, in execute
return meth(self, multiparams, params)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/sql/ddl.py", line 72, in _execute_on_connection
return connection._execute_ddl(self, multiparams, params)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1050, in _execute_ddl
compiled,
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1248, in _execute_context
e, statement, parameters, cursor, context
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1464, in _handle_dbapi_exception
util.raise_from_cause(newraise, exc_info)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 398, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 152, in reraise
raise value.with_traceback(tb)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1244, in _execute_context
cursor, statement, parameters, context
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 552, in do_execute
cursor.execute(statement, parameters)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/cursors.py", line 170, in execute
result = self._query(query)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/cursors.py", line 328, in _query
conn.query(q)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/connections.py", line 517, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/connections.py", line 732, in _read_query_result
result.read()
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/connections.py", line 1075, in read
first_packet = self.connection._read_packet()
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/connections.py", line 684, in _read_packet
packet.check_error()
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/protocol.py", line 220, in check_error
err.raise_mysql_exception(self._data)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/err.py", line 109, in raise_mysql_exception
raise errorclass(errno, errval)
oslo_db.exception.DBError: (pymysql.err.InternalError) (1832, "Cannot change column 'network_id': used in a foreign key constraint 'subnets_ibfk_1'")
[SQL: ALTER TABLE subnets MODIFY network_id VARCHAR(36) NOT NULL]
(Background on this error at: http://sqlalche.me/e/2j85)"
CentOS jobs are not affected. One major difference between these is that
Ubuntu uses Mariadb 10.1, whereas CentOS uses MariaDB 10.3.
Example:
https://storage.bhs1.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/logs_58/675658/8/check
/kolla-ansible-ubuntu-source/712c02b/primary/logs/ansible/deploy
** Affects: kolla
Importance: Critical
Status: New
** Affects: kolla/train
Importance: Critical
Status: New
** Affects: kolla-ansible
Importance: Critical
Status: New
** Affects: kolla-ansible/train
Importance: Critical
Status: New
** Affects: neutron
Importance: Undecided
Status: New
** Changed in: kolla-ansible
Importance: Undecided => Critical
** Also affects: kolla-ansible/train
Importance: Critical
Status: New
** Also affects: kolla
Importance: Undecided
Status: New
** Also affects: neutron
Importance: Undecided
Status: New
** Changed in: kolla
Importance: Undecided => Critical
** Also affects: kolla/train
Importance: Critical
Status: New
--
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to neutron.
https://bugs.launchpad.net/bugs/1841907
Title:
Neutron bootstrap failing on Ubuntu bionic with Cannot change column
'network_id
Status in kolla:
New
Status in kolla train series:
New
Status in kolla-ansible:
New
Status in kolla-ansible train series:
New
Status in neutron:
New
Bug description:
Neutron bootstrap is currently failing on Ubuntu bionic (kolla-
ansible-ubuntu-source jobs) with the following error:
INFO [alembic.runtime.migration] Running upgrade 63fd95af7dcd -> c613d0b82681
Traceback (most recent call last):
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1244, in _execute_context
cursor, statement, parameters, context
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 552, in do_execute
cursor.execute(statement, parameters)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/cursors.py", line 170, in execute
result = self._query(query)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/cursors.py", line 328, in _query
conn.query(q)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/connections.py", line 517, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/connections.py", line 732, in _read_query_result
result.read()
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/connections.py", line 1075, in read
first_packet = self.connection._read_packet()
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/connections.py", line 684, in _read_packet
packet.check_error()
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/protocol.py", line 220, in check_error
err.raise_mysql_exception(self._data)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/err.py", line 109, in raise_mysql_exception
raise errorclass(errno, errval)
pymysql.err.InternalError: (1832, "Cannot change column 'network_id': used in a foreign key constraint 'subnets_ibfk_1'")
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/var/lib/kolla/venv/bin/neutron-db-manage", line 10, in <module>
sys.exit(main())
File "/var/lib/kolla/venv/lib/python3.6/site-packages/neutron/db/migration/cli.py", line 657, in main
return_val |= bool(CONF.command.func(config, CONF.command.name))
File "/var/lib/kolla/venv/lib/python3.6/site-packages/neutron/db/migration/cli.py", line 181, in do_upgrade
desc=branch, sql=CONF.command.sql)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/neutron/db/migration/cli.py", line 82, in do_alembic_command
getattr(alembic_command, cmd)(config, *args, **kwargs)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/command.py", line 276, in upgrade
script.run_env()
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/script/base.py", line 475, in run_env
util.load_python_file(self.dir, "env.py")
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/util/pyfiles.py", line 90, in load_python_file
module = load_module_py(module_id, path)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/util/compat.py", line 177, in load_module_py
spec.loader.exec_module(module)
File "<frozen importlib._bootstrap_external>", line 678, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/var/lib/kolla/venv/lib/python3.6/site-packages/neutron/db/migration/alembic_migrations/env.py", line 120, in <module>
run_migrations_online()
File "/var/lib/kolla/venv/lib/python3.6/site-packages/neutron/db/migration/alembic_migrations/env.py", line 114, in run_migrations_online
context.run_migrations()
File "<string>", line 8, in run_migrations
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/runtime/environment.py", line 839, in run_migrations
self.get_context().run_migrations(**kw)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/runtime/migration.py", line 362, in run_migrations
step.migration_fn(**kw)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/neutron/db/migration/alembic_migrations/versions/train/expand/c613d0b82681_subnet_force_network_id.py", line 35, in upgrade
existing_type=sa.String(36))
File "<string>", line 8, in alter_column
File "<string>", line 3, in alter_column
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/operations/ops.py", line 1775, in alter_column
return operations.invoke(alt)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/operations/base.py", line 345, in invoke
return fn(self, operation)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/operations/toimpl.py", line 56, in alter_column
**operation.kw
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/ddl/mysql.py", line 96, in alter_column
else existing_comment,
File "/var/lib/kolla/venv/lib/python3.6/site-packages/alembic/ddl/impl.py", line 134, in _exec
return conn.execute(construct, *multiparams, **params)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 988, in execute
return meth(self, multiparams, params)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/sql/ddl.py", line 72, in _execute_on_connection
return connection._execute_ddl(self, multiparams, params)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1050, in _execute_ddl
compiled,
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1248, in _execute_context
e, statement, parameters, cursor, context
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1464, in _handle_dbapi_exception
util.raise_from_cause(newraise, exc_info)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 398, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 152, in reraise
raise value.with_traceback(tb)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1244, in _execute_context
cursor, statement, parameters, context
File "/var/lib/kolla/venv/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 552, in do_execute
cursor.execute(statement, parameters)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/cursors.py", line 170, in execute
result = self._query(query)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/cursors.py", line 328, in _query
conn.query(q)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/connections.py", line 517, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/connections.py", line 732, in _read_query_result
result.read()
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/connections.py", line 1075, in read
first_packet = self.connection._read_packet()
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/connections.py", line 684, in _read_packet
packet.check_error()
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/protocol.py", line 220, in check_error
err.raise_mysql_exception(self._data)
File "/var/lib/kolla/venv/lib/python3.6/site-packages/pymysql/err.py", line 109, in raise_mysql_exception
raise errorclass(errno, errval)
oslo_db.exception.DBError: (pymysql.err.InternalError) (1832, "Cannot change column 'network_id': used in a foreign key constraint 'subnets_ibfk_1'")
[SQL: ALTER TABLE subnets MODIFY network_id VARCHAR(36) NOT NULL]
(Background on this error at: http://sqlalche.me/e/2j85)"
CentOS jobs are not affected. One major difference between these is
that Ubuntu uses Mariadb 10.1, whereas CentOS uses MariaDB 10.3.
Example:
https://storage.bhs1.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/logs_58/675658/8/check
/kolla-ansible-ubuntu-source/712c02b/primary/logs/ansible/deploy
To manage notifications about this bug go to:
https://bugs.launchpad.net/kolla/+bug/1841907/+subscriptions
Follow ups
