yahoo-eng-team team mailing list archive

[Matt Riedemann <mriedem.os@xxxxxxxxx>]

Public bug reported:

http://logstash.openstack.org/#dashboard/file/logstash.json?query=message%3A%5C%22UserWarning%3A%20Cannot%20convert%20%3Coslo_db.sqlalchemy.enginefacade._Default%20object%20at%5C%22&from=7d

This shows up in nova functional test runs. This is the stacktrace from
one of the tests:

    2018-10-22 11:16:41,704 ERROR [root] Original exception being dropped: ['Traceback (most recent call last):\n', '  File "nova/exception_wrapper.py", line 69, in wrapped\n    return f(self, context, *args, **kw)\n', '  File "nova/compute/utils.py", line 1157, in decorated_function\n    return function(self, context, *args, **kwargs)\n', '  File "nova/compute/manager.py", line 216, in decorated_function\n    kwargs[\'instance\'], e, sys.exc_info())\n', '  File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__\n    self.force_reraise()\n', '  File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise\n    six.reraise(self.type_, self.value, self.tb)\n', '  File "nova/compute/manager.py", line 204, in decorated_function\n    return function(self, context, *args, **kwargs)\n', '  File "nova/compute/manager.py", line 6077, in check_can_live_migrate_destination\n    block_migration, disk_over_commit)\n', '  File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/mock/mock.py", line 1062, in __call__\n    return _mock_self._mock_call(*args, **kwargs)\n', '  File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/mock/mock.py", line 1128, in _mock_call\n    ret_val = effect(*args, **kwargs)\n', '  File "nova/tests/functional/test_servers.py", line 2864, in fake_check_can_live_migrate_destination\n    reason=\'test_live_migrate_pre_check_fails\')\n', 'MigrationPreCheckError: Migration pre-check error: test_live_migrate_pre_check_fails\n']
    2018-10-22 11:16:41,704 ERROR [oslo_messaging.rpc.server] Exception during message handling
    Traceback (most recent call last):
      File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_messaging/rpc/server.py", line 163, in _process_incoming
        res = self.dispatcher.dispatch(message)
      File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_messaging/rpc/dispatcher.py", line 265, in dispatch
        return self._do_dispatch(endpoint, method, ctxt, args)
      File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_messaging/rpc/dispatcher.py", line 194, in _do_dispatch
        result = func(ctxt, **new_args)
      File "nova/exception_wrapper.py", line 79, in wrapped
        function_name, call_dict, binary, tb)
      File "nova/exception_wrapper.py", line 33, in _emit_exception_notification
        args)
      File "nova/exception_wrapper.py", line 56, in _emit_legacy_exception_notification
        notifier.error(context, function_name, payload)
      File "nova/rpc.py", line 402, in _notify
        getattr(self.notifier, priority)(ctxt, event_type, payload)
      File "nova/tests/unit/fake_notifier.py", line 99, in _notify
        payload = self._serializer.serialize_entity(ctxt, payload)
      File "nova/rpc.py", line 134, in serialize_entity
        return self._base.serialize_entity(context, entity)
      File "nova/rpc.py", line 123, in serialize_entity
        return jsonutils.to_primitive(entity, convert_instances=True)
      File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in to_primitive
        for k, v in value.items()}
      File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in <dictcomp>
        for k, v in value.items()}
      File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in to_primitive
        for k, v in value.items()}
      File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in <dictcomp>
        for k, v in value.items()}
      File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 181, in to_primitive
        return recursive(value.__dict__, level=level + 1)
      File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in to_primitive
        for k, v in value.items()}
      File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in <dictcomp>
        for k, v in value.items()}
      File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 181, in to_primitive
        return recursive(value.__dict__, level=level + 1)
      File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in to_primitive
        for k, v in value.items()}
      File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in <dictcomp>
        for k, v in value.items()}
      File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 181, in to_primitive
        return recursive(value.__dict__, level=level + 1)
      File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in to_primitive
        for k, v in value.items()}
      File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in <dictcomp>
        for k, v in value.items()}
      File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in to_primitive
        for k, v in value.items()}
      File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in <dictcomp>
        for k, v in value.items()}
      File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 192, in to_primitive
        "instead of warning in version 3.0" % (value,))
    UserWarning: Cannot convert <oslo_db.sqlalchemy.enginefacade._Default object at 0x7f364c63f6a8> to primitive, will raise ValueError instead of warning in version 3.0

So there is something in the notification payload that has an instance
of oslo_db.sqlalchemy.enginefacade._Default in it and the serializer
doesn't know how to handle that. My guess would be there are versioned
objects in the payload and those versioned objects have a context which
has a db_connection engine facade in it, maybe because of using the
CheatingSerializer fixture in nova?

** Affects: nova
     Importance: Medium
         Status: Confirmed


** Tags: notifications rpc testing

-- 
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to OpenStack Compute (nova).
https://bugs.launchpad.net/bugs/1799249

Title:
  "UserWarning: Cannot convert <oslo_db.sqlalchemy.enginefacade._Default
  object at 0x7f5db695a2b8> to primitive, will raise ValueError instead
  of warning in version 3.0" seen in nova functional tests

Status in OpenStack Compute (nova):
  Confirmed

Bug description:
  http://logstash.openstack.org/#dashboard/file/logstash.json?query=message%3A%5C%22UserWarning%3A%20Cannot%20convert%20%3Coslo_db.sqlalchemy.enginefacade._Default%20object%20at%5C%22&from=7d

  This shows up in nova functional test runs. This is the stacktrace
  from one of the tests:

      2018-10-22 11:16:41,704 ERROR [root] Original exception being dropped: ['Traceback (most recent call last):\n', '  File "nova/exception_wrapper.py", line 69, in wrapped\n    return f(self, context, *args, **kw)\n', '  File "nova/compute/utils.py", line 1157, in decorated_function\n    return function(self, context, *args, **kwargs)\n', '  File "nova/compute/manager.py", line 216, in decorated_function\n    kwargs[\'instance\'], e, sys.exc_info())\n', '  File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__\n    self.force_reraise()\n', '  File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise\n    six.reraise(self.type_, self.value, self.tb)\n', '  File "nova/compute/manager.py", line 204, in decorated_function\n    return function(self, context, *args, **kwargs)\n', '  File "nova/compute/manager.py", line 6077, in check_can_live_migrate_destination\n    block_migration, disk_over_commit)\n', '  File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/mock/mock.py", line 1062, in __call__\n    return _mock_self._mock_call(*args, **kwargs)\n', '  File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/mock/mock.py", line 1128, in _mock_call\n    ret_val = effect(*args, **kwargs)\n', '  File "nova/tests/functional/test_servers.py", line 2864, in fake_check_can_live_migrate_destination\n    reason=\'test_live_migrate_pre_check_fails\')\n', 'MigrationPreCheckError: Migration pre-check error: test_live_migrate_pre_check_fails\n']
      2018-10-22 11:16:41,704 ERROR [oslo_messaging.rpc.server] Exception during message handling
      Traceback (most recent call last):
        File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_messaging/rpc/server.py", line 163, in _process_incoming
          res = self.dispatcher.dispatch(message)
        File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_messaging/rpc/dispatcher.py", line 265, in dispatch
          return self._do_dispatch(endpoint, method, ctxt, args)
        File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_messaging/rpc/dispatcher.py", line 194, in _do_dispatch
          result = func(ctxt, **new_args)
        File "nova/exception_wrapper.py", line 79, in wrapped
          function_name, call_dict, binary, tb)
        File "nova/exception_wrapper.py", line 33, in _emit_exception_notification
          args)
        File "nova/exception_wrapper.py", line 56, in _emit_legacy_exception_notification
          notifier.error(context, function_name, payload)
        File "nova/rpc.py", line 402, in _notify
          getattr(self.notifier, priority)(ctxt, event_type, payload)
        File "nova/tests/unit/fake_notifier.py", line 99, in _notify
          payload = self._serializer.serialize_entity(ctxt, payload)
        File "nova/rpc.py", line 134, in serialize_entity
          return self._base.serialize_entity(context, entity)
        File "nova/rpc.py", line 123, in serialize_entity
          return jsonutils.to_primitive(entity, convert_instances=True)
        File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in to_primitive
          for k, v in value.items()}
        File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in <dictcomp>
          for k, v in value.items()}
        File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in to_primitive
          for k, v in value.items()}
        File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in <dictcomp>
          for k, v in value.items()}
        File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 181, in to_primitive
          return recursive(value.__dict__, level=level + 1)
        File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in to_primitive
          for k, v in value.items()}
        File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in <dictcomp>
          for k, v in value.items()}
        File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 181, in to_primitive
          return recursive(value.__dict__, level=level + 1)
        File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in to_primitive
          for k, v in value.items()}
        File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in <dictcomp>
          for k, v in value.items()}
        File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 181, in to_primitive
          return recursive(value.__dict__, level=level + 1)
        File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in to_primitive
          for k, v in value.items()}
        File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in <dictcomp>
          for k, v in value.items()}
        File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in to_primitive
          for k, v in value.items()}
        File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 170, in <dictcomp>
          for k, v in value.items()}
        File "/home/osboxes/git/nova/.tox/py27/local/lib/python2.7/site-packages/oslo_serialization/jsonutils.py", line 192, in to_primitive
          "instead of warning in version 3.0" % (value,))
      UserWarning: Cannot convert <oslo_db.sqlalchemy.enginefacade._Default object at 0x7f364c63f6a8> to primitive, will raise ValueError instead of warning in version 3.0

  So there is something in the notification payload that has an instance
  of oslo_db.sqlalchemy.enginefacade._Default in it and the serializer
  doesn't know how to handle that. My guess would be there are versioned
  objects in the payload and those versioned objects have a context
  which has a db_connection engine facade in it, maybe because of using
  the CheatingSerializer fixture in nova?

To manage notifications about this bug go to:
https://bugs.launchpad.net/nova/+bug/1799249/+subscriptions