yahoo-eng-team team mailing list archive

Thread
Date
[Bug 1727708] [NEW] resize of vm with nfs volume fails with "Directory not empty: '/var/lib/nova/instances/ddd04ae5-6e02-4634-b8b1-8fd14d22773b"

To: yahoo-eng-team@xxxxxxxxxxxxxxxxxxx
From: Vladislav Belogrudov <vladislav.belogrudov@xxxxxxxxxx>
Date: Thu, 26 Oct 2017 11:58:37 -0000
Reply-to: Bug 1727708 <1727708@xxxxxxxxxxxxxxxxxx>
Sender: bounces@xxxxxxxxxxxxx
Public bug reported:

Nova Pike,

I have nfs for cinder volumes and ephemeral disks, /var/lib/nova/instances is a share mounted on compute hosts
I created a boot-able volume, ran an instance (no config-drives, etc). Initiated a resize from micro1 to micro2:

+--------------------------------------+-----------+-------+------+-----------+-------+-----------+
| ID                                   | Name      |   RAM | Disk | Ephemeral | VCPUs | Is Public |
+--------------------------------------+-----------+-------+------+-----------+-------+-----------+
| 80bd2ab5-d99d-4e0f-8ae9-26ee31534210 | m1.micro1 |    64 |    1 |         0 |     1 | True      |
| 23a09d59-a182-4863-9ecc-935de6c01170 | m1.micro2 |    65 |    2 |         1 |     1 | True      |

Before resize:

[root@operator ~]# ls -ltrah /var/nfs/*/*
-rw-r--r-- 1 root  root    93 Oct 26 12:30 /var/nfs/nova/compute_nodes
-rw-rw-rw- 1 42427 42427 1.0G Oct 26 12:47 /var/nfs/cinder/volume-a7b6fb81-8143-4bb3-9216-eba1e5dc0433

/var/nfs/nova/locks:
total 8.0K
-rw-r--r-- 1 root root    0 Oct 25 18:35 nova-storage-registry-lock
-rw-r--r-- 1 root root    0 Oct 26 11:03 nova-2dc09a364cbe47628fc413a564bc87425971543f
-rw-r--r-- 1 root root    0 Oct 26 12:18 nova-ephemeral_1_0706d66
-rw-r--r-- 1 root root    0 Oct 26 12:18 nova-swap_1
drwxr-xr-x 2 root root 4.0K Oct 26 12:18 .
drwxrwxrwx 5 root root 4.0K Oct 26 12:47 ..

/var/nfs/nova/_base:
total 20M
-rw-r--r-- 1 42427 42427  40M Oct 26 11:49 2dc09a364cbe47628fc413a564bc87425971543f
-rw-r--r-- 1 42427 42427 1.0G Oct 26 12:18 ephemeral_1_0706d66
drwxr-xr-x 2 root  root  4.0K Oct 26 12:18 .
-rw-r--r-- 1 42427 42427 1.0M Oct 26 12:18 swap_1
drwxrwxrwx 5 root  root  4.0K Oct 26 12:47 ..

/var/nfs/nova/14dba039-bc6a-4726-a582-a1c6cb7fddf4:
total 28K
drwxrwxrwx 5 root  root  4.0K Oct 26 12:47 ..
drwxr-xr-x 2 root  root  4.0K Oct 26 12:47 .
-rw-r--r-- 1 42427 42427  17K Oct 26 12:47 console.log

After resize:

[root@operator ~]# ls -ltrah /var/nfs/*/*
-rw-r--r-- 1 root  root    93 Oct 26 12:30 /var/nfs/nova/compute_nodes
-rw-rw-rw- 1 42427 42427 1.0G Oct 26 12:50 /var/nfs/cinder/volume-a7b6fb81-8143-4bb3-9216-eba1e5dc0433

/var/nfs/nova/locks:
total 8.0K
-rw-r--r-- 1 root root    0 Oct 25 18:35 nova-storage-registry-lock
-rw-r--r-- 1 root root    0 Oct 26 11:03 nova-2dc09a364cbe47628fc413a564bc87425971543f
-rw-r--r-- 1 root root    0 Oct 26 12:18 nova-ephemeral_1_0706d66
-rw-r--r-- 1 root root    0 Oct 26 12:18 nova-swap_1
drwxr-xr-x 2 root root 4.0K Oct 26 12:18 .
drwxrwxrwx 5 root root 4.0K Oct 26 12:49 ..

/var/nfs/nova/_base:
total 20M
-rw-r--r-- 1 42427 42427  40M Oct 26 11:49 2dc09a364cbe47628fc413a564bc87425971543f
drwxr-xr-x 2 root  root  4.0K Oct 26 12:18 .
-rw-r--r-- 1 42427 42427 1.0G Oct 26 12:49 ephemeral_1_0706d66
-rw-r--r-- 1 42427 42427 1.0M Oct 26 12:49 swap_1
drwxrwxrwx 5 root  root  4.0K Oct 26 12:49 ..

/var/nfs/nova/14dba039-bc6a-4726-a582-a1c6cb7fddf4:
total 420K
-rw-r--r-- 1 42427 42427 193K Oct 26 12:49 .nfs000000000038000e00000006
-rw-r--r-- 1 42427 42427 193K Oct 26 12:49 .nfs000000000038000f00000005
drwxrwxrwx 5 root  root  4.0K Oct 26 12:49 ..
-rw-r--r-- 1 42427 42427  20K Oct 26 12:49 .nfs000000000038001200000004
drwxr-xr-x 2 root  root  4.0K Oct 26 12:49 .

Error: Failed to perform requested operation on instance "test1", the
instance has an error status: Please try again later [Error: [Errno 39]
Directory not empty:
'/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4'].

nova compute log on original host:

2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [req-fff5cc2b-82a5-441b-a123-f0384f8db7e7 f19e34c10c0148509af54be1540a627a 62743901dffd4739be97e2d3a2deead7 - default default] [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] Setting instance vm_state to ERROR: OSError: [Errno 39] Directory not empty: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4'
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] Traceback (most recent call last):
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 6886, in _error_out_instance_on_exception
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]     yield
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 3609, in _confirm_resize
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]     network_info)
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 7727, in confirm_migration
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]     self._cleanup_resize(instance, network_info)
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 1149, in _cleanup_resize
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]     shutil.rmtree(inst_base)
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]   File "/usr/lib64/python2.7/shutil.py", line 256, in rmtree
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]     onerror(os.rmdir, path, sys.exc_info())
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]   File "/usr/lib64/python2.7/shutil.py", line 254, in rmtree
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]     os.rmdir(path)
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] OSError: [Errno 39] Directory not empty: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4'
2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server [req-fff5cc2b-82a5-441b-a123-f0384f8db7e7 f19e34c10c0148509af54be1540a627a 62743901dffd4739be97e2d3a2deead7 - default default] Exception during message handling: OSError: [Errno 39] Directory not empty: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4'
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server Traceback (most recent call last):
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_messaging/rpc/server.py", line 160, in _process_incoming
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     res = self.dispatcher.dispatch(message)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_messaging/rpc/dispatcher.py", line 213, in dispatch
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     return self._do_dispatch(endpoint, method, ctxt, args)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_messaging/rpc/dispatcher.py", line 183, in _do_dispatch
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     result = func(ctxt, **new_args)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/exception_wrapper.py", line 76, in wrapped
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     function_name, call_dict, binary)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     self.force_reraise()
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     six.reraise(self.type_, self.value, self.tb)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/exception_wrapper.py", line 67, in wrapped
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     return f(self, context, *args, **kw)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/utils.py", line 874, in decorated_function
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     return function(self, context, *args, **kwargs)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 217, in decorated_function
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     kwargs['instance'], e, sys.exc_info())
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     self.force_reraise()
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     six.reraise(self.type_, self.value, self.tb)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 205, in decorated_function
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     return function(self, context, *args, **kwargs)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 3587, in confirm_resize
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     do_confirm_resize(context, instance, migration.id)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_concurrency/lockutils.py", line 271, in inner
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     return f(*args, **kwargs)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 3585, in do_confirm_resize
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     self._confirm_resize(context, instance, migration=migration)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 3609, in _confirm_resize
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     network_info)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 7727, in confirm_migration
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     self._cleanup_resize(instance, network_info)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 1149, in _cleanup_resize
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     shutil.rmtree(inst_base)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/usr/lib64/python2.7/shutil.py", line 256, in rmtree
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     onerror(os.rmdir, path, sys.exc_info())
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/usr/lib64/python2.7/shutil.py", line 254, in rmtree
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     os.rmdir(path)
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server OSError: [Errno 39] Directory not empty: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4'
2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server
2017-10-26 12:50:27.547 1 WARNING nova.virt.libvirt.driver [req-b6000967-10d8-4ad2-939e-44509ea6f900 - - - - -] Periodic task is updating the host stat, it is trying to get disk instance-00000005, but disk file was removed by concurrent operations such as resize.: OSError: [Errno 2] No such file or directory: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4/disk.local'
2017-10-26 12:50:28.048 1 WARNING nova.compute.resource_tracker [req-b6000967-10d8-4ad2-939e-44509ea6f900 - - - - -] [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] Instance not resizing, skipping migration.

The same thing works with cinder lvm backend quite well, also ephemeral
instances can be resized from micro1 to micro2.

** Affects: nova
     Importance: Undecided
         Status: New

** Description changed:

  Nova Pike,
  
  I have nfs for cinder volumes and ephemeral disks, /var/lib/nova/instances is a share mounted on compute hosts
  I created a boot-able volume, ran an instance (no config-drives, etc). Initiated a resize from micro1 to micro2:
  
  +--------------------------------------+-----------+-------+------+-----------+-------+-----------+
  | ID                                   | Name      |   RAM | Disk | Ephemeral | VCPUs | Is Public |
  +--------------------------------------+-----------+-------+------+-----------+-------+-----------+
  | 80bd2ab5-d99d-4e0f-8ae9-26ee31534210 | m1.micro1 |    64 |    1 |         0 |     1 | True      |
  | 23a09d59-a182-4863-9ecc-935de6c01170 | m1.micro2 |    65 |    2 |         1 |     1 | True      |
- 
  
  Before resize:
  
  [root@operator ~]# ls -ltrah /var/nfs/*/*
  -rw-r--r-- 1 root  root    93 Oct 26 12:30 /var/nfs/nova/compute_nodes
  -rw-rw-rw- 1 42427 42427 1.0G Oct 26 12:47 /var/nfs/cinder/volume-a7b6fb81-8143-4bb3-9216-eba1e5dc0433
  
  /var/nfs/nova/locks:
  total 8.0K
  -rw-r--r-- 1 root root    0 Oct 25 18:35 nova-storage-registry-lock
  -rw-r--r-- 1 root root    0 Oct 26 11:03 nova-2dc09a364cbe47628fc413a564bc87425971543f
  -rw-r--r-- 1 root root    0 Oct 26 12:18 nova-ephemeral_1_0706d66
  -rw-r--r-- 1 root root    0 Oct 26 12:18 nova-swap_1
  drwxr-xr-x 2 root root 4.0K Oct 26 12:18 .
  drwxrwxrwx 5 root root 4.0K Oct 26 12:47 ..
  
  /var/nfs/nova/_base:
  total 20M
  -rw-r--r-- 1 42427 42427  40M Oct 26 11:49 2dc09a364cbe47628fc413a564bc87425971543f
  -rw-r--r-- 1 42427 42427 1.0G Oct 26 12:18 ephemeral_1_0706d66
  drwxr-xr-x 2 root  root  4.0K Oct 26 12:18 .
  -rw-r--r-- 1 42427 42427 1.0M Oct 26 12:18 swap_1
  drwxrwxrwx 5 root  root  4.0K Oct 26 12:47 ..
  
  /var/nfs/nova/14dba039-bc6a-4726-a582-a1c6cb7fddf4:
  total 28K
  drwxrwxrwx 5 root  root  4.0K Oct 26 12:47 ..
  drwxr-xr-x 2 root  root  4.0K Oct 26 12:47 .
  -rw-r--r-- 1 42427 42427  17K Oct 26 12:47 console.log
- 
  
  After resize:
  
  [root@operator ~]# ls -ltrah /var/nfs/*/*
  -rw-r--r-- 1 root  root    93 Oct 26 12:30 /var/nfs/nova/compute_nodes
  -rw-rw-rw- 1 42427 42427 1.0G Oct 26 12:50 /var/nfs/cinder/volume-a7b6fb81-8143-4bb3-9216-eba1e5dc0433
  
  /var/nfs/nova/locks:
  total 8.0K
  -rw-r--r-- 1 root root    0 Oct 25 18:35 nova-storage-registry-lock
  -rw-r--r-- 1 root root    0 Oct 26 11:03 nova-2dc09a364cbe47628fc413a564bc87425971543f
  -rw-r--r-- 1 root root    0 Oct 26 12:18 nova-ephemeral_1_0706d66
  -rw-r--r-- 1 root root    0 Oct 26 12:18 nova-swap_1
  drwxr-xr-x 2 root root 4.0K Oct 26 12:18 .
  drwxrwxrwx 5 root root 4.0K Oct 26 12:49 ..
  
  /var/nfs/nova/_base:
  total 20M
  -rw-r--r-- 1 42427 42427  40M Oct 26 11:49 2dc09a364cbe47628fc413a564bc87425971543f
  drwxr-xr-x 2 root  root  4.0K Oct 26 12:18 .
  -rw-r--r-- 1 42427 42427 1.0G Oct 26 12:49 ephemeral_1_0706d66
  -rw-r--r-- 1 42427 42427 1.0M Oct 26 12:49 swap_1
  drwxrwxrwx 5 root  root  4.0K Oct 26 12:49 ..
  
  /var/nfs/nova/14dba039-bc6a-4726-a582-a1c6cb7fddf4:
  total 420K
  -rw-r--r-- 1 42427 42427 193K Oct 26 12:49 .nfs000000000038000e00000006
  -rw-r--r-- 1 42427 42427 193K Oct 26 12:49 .nfs000000000038000f00000005
  drwxrwxrwx 5 root  root  4.0K Oct 26 12:49 ..
  -rw-r--r-- 1 42427 42427  20K Oct 26 12:49 .nfs000000000038001200000004
  drwxr-xr-x 2 root  root  4.0K Oct 26 12:49 .
  
- 
- 
- Error: Failed to perform requested operation on instance "test1", the instance has an error status: Please try again later [Error: [Errno 39] Directory not empty: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4']. 
- 
+ Error: Failed to perform requested operation on instance "test1", the
+ instance has an error status: Please try again later [Error: [Errno 39]
+ Directory not empty:
+ '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4'].
  
  nova compute log on original host:
  
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [req-fff5cc2b-82a5-441b-a123-f0384f8db7e7 f19e34c10c0148509af54be1540a627a 62743901dffd4739be97e2d3a2deead7 - default default] [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] Setting instance vm_state to ERROR: OSError: [Errno 39] Directory not empty: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4'
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] Traceback (most recent call last):
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 6886, in _error_out_instance_on_exception
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]     yield
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 3609, in _confirm_resize
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]     network_info)
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 7727, in confirm_migration
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]     self._cleanup_resize(instance, network_info)
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 1149, in _cleanup_resize
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]     shutil.rmtree(inst_base)
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]   File "/usr/lib64/python2.7/shutil.py", line 256, in rmtree
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]     onerror(os.rmdir, path, sys.exc_info())
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]   File "/usr/lib64/python2.7/shutil.py", line 254, in rmtree
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]     os.rmdir(path)
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] OSError: [Errno 39] Directory not empty: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4'
- 2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] 
+ 2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server [req-fff5cc2b-82a5-441b-a123-f0384f8db7e7 f19e34c10c0148509af54be1540a627a 62743901dffd4739be97e2d3a2deead7 - default default] Exception during message handling: OSError: [Errno 39] Directory not empty: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4'
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server Traceback (most recent call last):
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_messaging/rpc/server.py", line 160, in _process_incoming
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     res = self.dispatcher.dispatch(message)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_messaging/rpc/dispatcher.py", line 213, in dispatch
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     return self._do_dispatch(endpoint, method, ctxt, args)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_messaging/rpc/dispatcher.py", line 183, in _do_dispatch
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     result = func(ctxt, **new_args)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/exception_wrapper.py", line 76, in wrapped
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     function_name, call_dict, binary)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     self.force_reraise()
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     six.reraise(self.type_, self.value, self.tb)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/exception_wrapper.py", line 67, in wrapped
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     return f(self, context, *args, **kw)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/utils.py", line 874, in decorated_function
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     return function(self, context, *args, **kwargs)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 217, in decorated_function
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     kwargs['instance'], e, sys.exc_info())
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     self.force_reraise()
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     six.reraise(self.type_, self.value, self.tb)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 205, in decorated_function
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     return function(self, context, *args, **kwargs)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 3587, in confirm_resize
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     do_confirm_resize(context, instance, migration.id)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_concurrency/lockutils.py", line 271, in inner
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     return f(*args, **kwargs)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 3585, in do_confirm_resize
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     self._confirm_resize(context, instance, migration=migration)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 3609, in _confirm_resize
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     network_info)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 7727, in confirm_migration
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     self._cleanup_resize(instance, network_info)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 1149, in _cleanup_resize
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     shutil.rmtree(inst_base)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/usr/lib64/python2.7/shutil.py", line 256, in rmtree
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     onerror(os.rmdir, path, sys.exc_info())
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/usr/lib64/python2.7/shutil.py", line 254, in rmtree
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     os.rmdir(path)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server OSError: [Errno 39] Directory not empty: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4'
- 2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server 
+ 2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server
  2017-10-26 12:50:27.547 1 WARNING nova.virt.libvirt.driver [req-b6000967-10d8-4ad2-939e-44509ea6f900 - - - - -] Periodic task is updating the host stat, it is trying to get disk instance-00000005, but disk file was removed by concurrent operations such as resize.: OSError: [Errno 2] No such file or directory: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4/disk.local'
  2017-10-26 12:50:28.048 1 WARNING nova.compute.resource_tracker [req-b6000967-10d8-4ad2-939e-44509ea6f900 - - - - -] [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] Instance not resizing, skipping migration.
  
- 
- The same thing works with cinder lvm backend quite well
+ The same thing works with cinder lvm backend quite well, also ephemeral
+ instances can be resized from micro1 to micro2.

-- 
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to OpenStack Compute (nova).
https://bugs.launchpad.net/bugs/1727708

Title:
  resize of vm with nfs volume fails with  "Directory not empty:
  '/var/lib/nova/instances/ddd04ae5-6e02-4634-b8b1-8fd14d22773b"

Status in OpenStack Compute (nova):
  New

Bug description:
  Nova Pike,

  I have nfs for cinder volumes and ephemeral disks, /var/lib/nova/instances is a share mounted on compute hosts
  I created a boot-able volume, ran an instance (no config-drives, etc). Initiated a resize from micro1 to micro2:

  +--------------------------------------+-----------+-------+------+-----------+-------+-----------+
  | ID                                   | Name      |   RAM | Disk | Ephemeral | VCPUs | Is Public |
  +--------------------------------------+-----------+-------+------+-----------+-------+-----------+
  | 80bd2ab5-d99d-4e0f-8ae9-26ee31534210 | m1.micro1 |    64 |    1 |         0 |     1 | True      |
  | 23a09d59-a182-4863-9ecc-935de6c01170 | m1.micro2 |    65 |    2 |         1 |     1 | True      |

  Before resize:

  [root@operator ~]# ls -ltrah /var/nfs/*/*
  -rw-r--r-- 1 root  root    93 Oct 26 12:30 /var/nfs/nova/compute_nodes
  -rw-rw-rw- 1 42427 42427 1.0G Oct 26 12:47 /var/nfs/cinder/volume-a7b6fb81-8143-4bb3-9216-eba1e5dc0433

  /var/nfs/nova/locks:
  total 8.0K
  -rw-r--r-- 1 root root    0 Oct 25 18:35 nova-storage-registry-lock
  -rw-r--r-- 1 root root    0 Oct 26 11:03 nova-2dc09a364cbe47628fc413a564bc87425971543f
  -rw-r--r-- 1 root root    0 Oct 26 12:18 nova-ephemeral_1_0706d66
  -rw-r--r-- 1 root root    0 Oct 26 12:18 nova-swap_1
  drwxr-xr-x 2 root root 4.0K Oct 26 12:18 .
  drwxrwxrwx 5 root root 4.0K Oct 26 12:47 ..

  /var/nfs/nova/_base:
  total 20M
  -rw-r--r-- 1 42427 42427  40M Oct 26 11:49 2dc09a364cbe47628fc413a564bc87425971543f
  -rw-r--r-- 1 42427 42427 1.0G Oct 26 12:18 ephemeral_1_0706d66
  drwxr-xr-x 2 root  root  4.0K Oct 26 12:18 .
  -rw-r--r-- 1 42427 42427 1.0M Oct 26 12:18 swap_1
  drwxrwxrwx 5 root  root  4.0K Oct 26 12:47 ..

  /var/nfs/nova/14dba039-bc6a-4726-a582-a1c6cb7fddf4:
  total 28K
  drwxrwxrwx 5 root  root  4.0K Oct 26 12:47 ..
  drwxr-xr-x 2 root  root  4.0K Oct 26 12:47 .
  -rw-r--r-- 1 42427 42427  17K Oct 26 12:47 console.log

  After resize:

  [root@operator ~]# ls -ltrah /var/nfs/*/*
  -rw-r--r-- 1 root  root    93 Oct 26 12:30 /var/nfs/nova/compute_nodes
  -rw-rw-rw- 1 42427 42427 1.0G Oct 26 12:50 /var/nfs/cinder/volume-a7b6fb81-8143-4bb3-9216-eba1e5dc0433

  /var/nfs/nova/locks:
  total 8.0K
  -rw-r--r-- 1 root root    0 Oct 25 18:35 nova-storage-registry-lock
  -rw-r--r-- 1 root root    0 Oct 26 11:03 nova-2dc09a364cbe47628fc413a564bc87425971543f
  -rw-r--r-- 1 root root    0 Oct 26 12:18 nova-ephemeral_1_0706d66
  -rw-r--r-- 1 root root    0 Oct 26 12:18 nova-swap_1
  drwxr-xr-x 2 root root 4.0K Oct 26 12:18 .
  drwxrwxrwx 5 root root 4.0K Oct 26 12:49 ..

  /var/nfs/nova/_base:
  total 20M
  -rw-r--r-- 1 42427 42427  40M Oct 26 11:49 2dc09a364cbe47628fc413a564bc87425971543f
  drwxr-xr-x 2 root  root  4.0K Oct 26 12:18 .
  -rw-r--r-- 1 42427 42427 1.0G Oct 26 12:49 ephemeral_1_0706d66
  -rw-r--r-- 1 42427 42427 1.0M Oct 26 12:49 swap_1
  drwxrwxrwx 5 root  root  4.0K Oct 26 12:49 ..

  /var/nfs/nova/14dba039-bc6a-4726-a582-a1c6cb7fddf4:
  total 420K
  -rw-r--r-- 1 42427 42427 193K Oct 26 12:49 .nfs000000000038000e00000006
  -rw-r--r-- 1 42427 42427 193K Oct 26 12:49 .nfs000000000038000f00000005
  drwxrwxrwx 5 root  root  4.0K Oct 26 12:49 ..
  -rw-r--r-- 1 42427 42427  20K Oct 26 12:49 .nfs000000000038001200000004
  drwxr-xr-x 2 root  root  4.0K Oct 26 12:49 .

  Error: Failed to perform requested operation on instance "test1", the
  instance has an error status: Please try again later [Error: [Errno
  39] Directory not empty:
  '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4'].

  nova compute log on original host:

  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [req-fff5cc2b-82a5-441b-a123-f0384f8db7e7 f19e34c10c0148509af54be1540a627a 62743901dffd4739be97e2d3a2deead7 - default default] [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] Setting instance vm_state to ERROR: OSError: [Errno 39] Directory not empty: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4'
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] Traceback (most recent call last):
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 6886, in _error_out_instance_on_exception
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]     yield
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 3609, in _confirm_resize
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]     network_info)
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 7727, in confirm_migration
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]     self._cleanup_resize(instance, network_info)
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 1149, in _cleanup_resize
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]     shutil.rmtree(inst_base)
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]   File "/usr/lib64/python2.7/shutil.py", line 256, in rmtree
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]     onerror(os.rmdir, path, sys.exc_info())
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]   File "/usr/lib64/python2.7/shutil.py", line 254, in rmtree
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]     os.rmdir(path)
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] OSError: [Errno 39] Directory not empty: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4'
  2017-10-26 12:49:50.267 1 ERROR nova.compute.manager [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4]
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server [req-fff5cc2b-82a5-441b-a123-f0384f8db7e7 f19e34c10c0148509af54be1540a627a 62743901dffd4739be97e2d3a2deead7 - default default] Exception during message handling: OSError: [Errno 39] Directory not empty: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4'
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server Traceback (most recent call last):
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_messaging/rpc/server.py", line 160, in _process_incoming
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     res = self.dispatcher.dispatch(message)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_messaging/rpc/dispatcher.py", line 213, in dispatch
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     return self._do_dispatch(endpoint, method, ctxt, args)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_messaging/rpc/dispatcher.py", line 183, in _do_dispatch
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     result = func(ctxt, **new_args)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/exception_wrapper.py", line 76, in wrapped
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     function_name, call_dict, binary)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     self.force_reraise()
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     six.reraise(self.type_, self.value, self.tb)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/exception_wrapper.py", line 67, in wrapped
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     return f(self, context, *args, **kw)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/utils.py", line 874, in decorated_function
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     return function(self, context, *args, **kwargs)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 217, in decorated_function
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     kwargs['instance'], e, sys.exc_info())
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     self.force_reraise()
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     six.reraise(self.type_, self.value, self.tb)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 205, in decorated_function
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     return function(self, context, *args, **kwargs)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 3587, in confirm_resize
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     do_confirm_resize(context, instance, migration.id)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/oslo_concurrency/lockutils.py", line 271, in inner
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     return f(*args, **kwargs)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 3585, in do_confirm_resize
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     self._confirm_resize(context, instance, migration=migration)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/compute/manager.py", line 3609, in _confirm_resize
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     network_info)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 7727, in confirm_migration
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     self._cleanup_resize(instance, network_info)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/var/lib/kolla/venv/lib/python2.7/site-packages/nova/virt/libvirt/driver.py", line 1149, in _cleanup_resize
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     shutil.rmtree(inst_base)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/usr/lib64/python2.7/shutil.py", line 256, in rmtree
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     onerror(os.rmdir, path, sys.exc_info())
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server   File "/usr/lib64/python2.7/shutil.py", line 254, in rmtree
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server     os.rmdir(path)
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server OSError: [Errno 39] Directory not empty: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4'
  2017-10-26 12:49:50.474 1 ERROR oslo_messaging.rpc.server
  2017-10-26 12:50:27.547 1 WARNING nova.virt.libvirt.driver [req-b6000967-10d8-4ad2-939e-44509ea6f900 - - - - -] Periodic task is updating the host stat, it is trying to get disk instance-00000005, but disk file was removed by concurrent operations such as resize.: OSError: [Errno 2] No such file or directory: '/var/lib/nova/instances/14dba039-bc6a-4726-a582-a1c6cb7fddf4/disk.local'
  2017-10-26 12:50:28.048 1 WARNING nova.compute.resource_tracker [req-b6000967-10d8-4ad2-939e-44509ea6f900 - - - - -] [instance: 14dba039-bc6a-4726-a582-a1c6cb7fddf4] Instance not resizing, skipping migration.

  The same thing works with cinder lvm backend quite well, also
  ephemeral instances can be resized from micro1 to micro2.

To manage notifications about this bug go to:
https://bugs.launchpad.net/nova/+bug/1727708/+subscriptions