openstack team mailing list archive

[Vishvananda Ishaya <vishvananda@xxxxxxxxx>]

Hey everyone,

I'm a bit concerned with the connection pooling in the db.  It seems that things are not getting cleaned up properly.  I have a repro-case that causes failures that we have seen before.  if I revert the nova/db/sqlalchemy/session.py to before the eventlet db pool was added I get no failures.  If you want to see the issue, try the attached code.  You will need to run from the nova directory or do python setup.py develop.  You will also need to create a mysql database called test and edit the sql_connection string if you have a mysql password, etc.  Please check this code.  If we can't come up with a fix, I htink we need to revert back to no connection pooling.

Run the attached script at least 3 times The code below runs fine the first couple of times, Then it starts to fail with the following error:

2011-09-24 12:51:02,799 INFO sqlalchemy.engine.base.Engine.0x...36d0 [-] ROLLBACK
Traceback (most recent call last):
 File "/Library/Python/2.7/site-packages/eventlet/hubs/hub.py", line 336, in fire_timers
   timer()
 File "/Library/Python/2.7/site-packages/eventlet/hubs/timer.py", line 56, in __call__
   cb(*args, **kw)
 File "/Library/Python/2.7/site-packages/eventlet/event.py", line 163, in _do_send
   waiter.switch(result)
 File "/Library/Python/2.7/site-packages/eventlet/greenthread.py", line 192, in main
   result = function(*args, **kwargs)
 File "dbrepro.py", line 44, in associate
   ip = db.fixed_ip_associate_pool(ctxt, 1, instance_id=val)
 File "/Users/vishvananda/os/nova/nova/db/api.py", line 352, in fixed_ip_associate_pool
   instance_id, host)
 File "/Users/vishvananda/os/nova/nova/db/sqlalchemy/api.py", line 102, in wrapper
   return f(*args, **kwargs)
 File "/Users/vishvananda/os/nova/nova/db/sqlalchemy/api.py", line 725, in fixed_ip_associate_pool
   filter_by(host=None).\
 File "/Library/Python/2.7/site-packages/sqlalchemy/orm/query.py", line 1496, in first
   ret = list(self[0:1])
 File "/Library/Python/2.7/site-packages/sqlalchemy/orm/query.py", line 1405, in __getitem__
   return list(res)
 File "/Library/Python/2.7/site-packages/sqlalchemy/orm/query.py", line 1669, in instances
   fetch = cursor.fetchall()
 File "/Library/Python/2.7/site-packages/sqlalchemy/engine/base.py", line 2383, in fetchall
   l = self.process_rows(self._fetchall_impl())
 File "/Library/Python/2.7/site-packages/sqlalchemy/engine/base.py", line 2366, in process_rows
   keymap = metadata._keymap
AttributeError: 'NoneType' object has no attribute '_keymap'

import eventlet
eventlet.monkey_patch()

from nova import context
from nova import db
from nova import flags
from nova import log as logging
from nova.db import migration
from nova.network import manager as network_manager

from sqlalchemy import exc
FLAGS = flags.FLAGS

FLAGS.sql_connection = 'mysql://root:@localhost/test'
from nova.tests import fake_flags
logging.setup()
ctxt = context.get_admin_context()

def setup():
    migration.db_sync()
    network = network_manager.VlanManager()
    bridge_interface = FLAGS.flat_interface or FLAGS.vlan_interface
    network.create_networks(ctxt,
                            label='test',
                            cidr=FLAGS.fixed_range,
                            multi_host=FLAGS.multi_host,
                            num_networks=FLAGS.num_networks,
                            network_size=FLAGS.network_size,
                            cidr_v6=FLAGS.fixed_range_v6,
                            gateway_v6=FLAGS.gateway_v6,
                            bridge=FLAGS.flat_network_bridge,
                            bridge_interface=bridge_interface,
                            vpn_start=FLAGS.vpn_start,
                            vlan_start=FLAGS.vlan_start,
                            dns1=FLAGS.flat_network_dns)
    for net in db.network_get_all(ctxt):
        network.set_network_host(ctxt, net)
    for i in range(5):
        db.instance_create(ctxt, {})



import time
def associate(val):
    ip = db.fixed_ip_associate_pool(ctxt, 1, instance_id=val)
    time.sleep(1.0)
    db.fixed_ip_disassociate(ctxt, ip)
    return ip

try:
    result = db.fixed_ip_get_all(ctxt)
except exc.ProgrammingError:
    setup()

pile = eventlet.GreenPile()
jobs = []
for i in range(5):
    pile.spawn(associate, i)
    #p = multiprocessing.Process(target=associate, args=(i,))
    #jobs.append(p)
    #p.start()

for result in pile:
    print result