yahoo-eng-team team mailing list archive

[tangyi <tinetkiller@xxxxxxxxx>]

Public bug reported:

 vm occur i/o blocked about every five days. vm log as below:

INFO: task jbd2/dm-0-8:373 blocked for more than 120 seconds. Not
tainted 2.6.32-504.30.3.el6.x86_64 #1 "echo 0 >
/proc/sys/kernel/hung_task_timeout_secs" disables this message.
jbd2/dm-0-8 D 0000000000000000 0 373 2 0x00000000 ffff880037f5bd20
0000000000000046 00000000000158c0 00000000000158c0 ffff88000220f980
00000000000158c0 00000000000158c0 ffff880037e48040 ffffffff81a8d020
ffffffff8160dd00 ffff880037e485f8 ffff880037f5bfd8 Call Trace:
[<ffffffff8109ef4e>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa007480f>]
jbd2_journal_commit_transaction+0x19f/0x1500 [jbd2] [<ffffffff8108742c>]
? lock_timer_base+0x3c/0x70 [<ffffffff8109ec20>] ?
autoremove_wake_function+0x0/0x40 [<ffffffffa007aa48>]
kjournald2+0xb8/0x220 [jbd2] [<ffffffff8109ec20>] ?
autoremove_wake_function+0x0/0x40 [<ffffffffa007a990>] ?
kjournald2+0x0/0x220 [jbd2] [<ffffffff8109e78e>] kthread+0x9e/0xc0
[<ffffffff8100c28a>] child_rip+0xa/0x20 [<ffffffff8109e6f0>] ?
kthread+0x0/0xc0 [<ffffffff8100c280>] ? child_rip+0x0/0x20 INFO: task
auditd:978 blocked for more than 120 seconds. Not tainted
2.6.32-504.30.3.el6.x86_64 #1 "echo 0 >
/proc/sys/kernel/hung_task_timeout_secs" disables this message. auditd D
0000000000000000 0 978 1 0x00000000 ffff88007c261a88 0000000000000086
0000000000000000 ffff88002d1993d8 ffff88002d1993d8 ffff880079cbf4e8
00035e86aaf88734 0000000000000000 ffff88007c261a28 0000000138823544
ffff8800379365f8 ffff88007c261fd8 Call Trace: [<ffffffffa007308a>]
start_this_handle+0x25a/0x480 [jbd2] [<ffffffff8109ec20>] ?
autoremove_wake_function+0x0/0x40 [<ffffffffa0073495>]
jbd2_journal_start+0xb5/0x100 [jbd2] [<ffffffffa00b2e66>]
ext4_journal_start_sb+0x56/0xe0 [ext4] [<ffffffffa009bf0a>]
ext4_dirty_inode+0x2a/0x60 [ext4] [<ffffffff811baa9b>]
__mark_inode_dirty+0x3b/0x160 [<ffffffff811ab182>]
file_update_time+0xf2/0x170 [<ffffffff811268a0>]
__generic_file_aio_write+0x230/0x490 [<ffffffff81126b88>]
generic_file_aio_write+0x88/0x100 [<ffffffffa0095e08>]
ext4_file_write+0x58/0x190 [ext4] [<ffffffff8118e0aa>]
do_sync_write+0xfa/0x140 [<ffffffffa007a5d5>] ?
jbd2_log_wait_commit+0xf5/0x140 [jbd2] [<ffffffffa00b004f>] ?
ext4_statfs+0xef/0x200 [ext4] [<ffffffff8109ec20>] ?
autoremove_wake_function+0x0/0x40 [<ffffffff811c2f38>] ?
do_statfs_native+0x98/0xb0 [<ffffffff8122da86>] ?
security_file_permission+0x16/0x20 [<ffffffff8118e3a8>]
vfs_write+0xb8/0x1a0 [<ffffffff8118ed71>] sys_write+0x51/0x90
[<ffffffff8100b0d2>] system_call_fastpath+0x16/0x1b INFO: task rs:main
Q:Reg:1017 blocked for more than 120 seconds. Not tainted
2.6.32-504.30.3.el6.x86_64 #1 "echo 0 >
/proc/sys/kernel/hung_task_timeout_secs" disables this message. rs:main
Q:Reg D 0000000000000000 0 1017 1 0x00000080 ffff88007a5f3a88
0000000000000086 0000000000000000 00000002000093d8 0000000000000000
ffff88000220f660 00035e86aa672ebb 0000000000000008 00000000000158c0
0000000138823478 ffff880037d01068 ffff88007a5f3fd8 Call Trace:
[<ffffffffa007308a>] start_this_handle+0x25a/0x480 [jbd2]
[<ffffffff81173fbb>] ? cache_alloc_refill+0x15b/0x240
[<ffffffff8109ec20>] ? autoremove_wake_function+0x0/0x40
[<ffffffffa0073495>] jbd2_journal_start+0xb5/0x100 [jbd2]
[<ffffffffa00b2e66>] ext4_journal_start_sb+0x56/0xe0 [ext4]
[<ffffffffa009bf0a>] ext4_dirty_inode+0x2a/0x60 [ext4]
[<ffffffff811baa9b>] __mark_inode_dirty+0x3b/0x160 [<ffffffff811ab182>]
file_update_time+0xf2/0x170 [<ffffffff811268a0>]
__generic_file_aio_write+0x230/0x490 [<ffffffff81126b88>]
generic_file_aio_write+0x88/0x100 [<ffffffffa0095e08>]
ext4_file_write+0x58/0x190 [ext4] [<ffffffff8118e0aa>]
do_sync_write+0xfa/0x140 [<ffffffff8109ec20>] ?
autoremove_wake_function+0x0/0x40 [<ffffffff8122da86>] ?
security_file_permission+0x16/0x20 [<ffffffff8118e3a8>]
vfs_write+0xb8/0x1a0 [<ffffffff8118ed71>] sys_write+0x51/0x90
[<ffffffff810e5c5e>] ? __audit_syscall_exit+0x25e/0x290
[<ffffffff8100b0d2>] system_call_fastpath+0x16/0x1b INFO: task
flush-253:0:1016 blocked for more than 120 seconds. Not tainted
2.6.32-504.30.3.el6.x86_64 #1 "echo 0 >
/proc/sys/kernel/hung_task_timeout_secs" disables this message.
flush-253:0 D 0000000000000000 0 1016 2 0x00000080 ffff880037fbb430
0000000000000046 0000000000000000 ffffffff81041e98 ffff8800ffffffff
00000000092625a3 00035e80853b5e12 ffff880037dcb910 000000000b5a83fa
000000013881ce2a ffff880037b19ad8 ffff880037fbbfd8 Call Trace:
[<ffffffff81041e98>] ? pvclock_clocksource_read+0x58/0xd0
[<ffffffff811c5310>] ? sync_buffer+0x0/0x50 [<ffffffff8152a2f3>]
io_schedule+0x73/0xc0 [<ffffffff811c5350>] sync_buffer+0x40/0x50
[<ffffffff8152ab8a>] __wait_on_bit_lock+0x5a/0xc0 [<ffffffff811c5310>] ?
sync_buffer+0x0/0x50 [<ffffffff8152ac68>]
out_of_line_wait_on_bit_lock+0x78/0x90 [<ffffffff8109eca0>] ?
wake_bit_function+0x0/0x50 [<ffffffff811c48a9>] ?
__find_get_block+0xa9/0x200 [<ffffffff811c54f6>] __lock_buffer+0x36/0x40
[<ffffffffa0074293>] do_get_write_access+0x493/0x520 [jbd2]
[<ffffffffa0074471>] jbd2_journal_get_write_access+0x31/0x50 [jbd2]
[<ffffffffa00c1da8>] __ext4_journal_get_write_access+0x38/0x80 [ext4]
[<ffffffffa009bbb3>] ext4_reserve_inode_write+0x73/0xa0 [ext4]
[<ffffffffa007378f>] ? jbd2_journal_dirty_metadata+0xff/0x150 [jbd2]
[<ffffffffa009bc2c>] ext4_mark_inode_dirty+0x4c/0x1d0 [ext4]
[<ffffffffa009bf20>] ext4_dirty_inode+0x40/0x60 [ext4]
[<ffffffff811baa9b>] __mark_inode_dirty+0x3b/0x160 [<ffffffffa009b241>]
ext4_da_update_reserve_space+0x111/0x2a0 [ext4] [<ffffffffa00bf97d>]
ext4_ext_get_blocks+0x72d/0x14d0 [ext4] [<ffffffff81270c90>] ?
generic_make_request+0x240/0x5a0 [<ffffffff81126c85>] ?
mempool_alloc_slab+0x15/0x20 [<ffffffffa009cbd9>]
ext4_get_blocks+0xf9/0x2b0 [ext4] [<ffffffff8113a7e5>] ?
pagevec_lookup_tag+0x25/0x40 [<ffffffffa00a1211>]
mpage_da_map_and_submit+0xa1/0x470 [ext4] [<ffffffffa0073495>] ?
jbd2_journal_start+0xb5/0x100 [jbd2] [<ffffffffa00a1e2e>]
ext4_da_writepages+0x2ee/0x620 [ext4] [<ffffffff81139871>]
do_writepages+0x21/0x40 [<ffffffff811bb19d>]
writeback_single_inode+0xdd/0x290 [<ffffffff811bb59d>]
writeback_sb_inodes+0xbd/0x170 [<ffffffff811bb6fb>]
writeback_inodes_wb+0xab/0x1b0 [<ffffffff811bbaf3>]
wb_writeback+0x2f3/0x410 [<ffffffff810880b2>] ? del_timer_sync+0x22/0x30
[<ffffffff811bbdb5>] wb_do_writeback+0x1a5/0x240 [<ffffffff811bbeb3>]
bdi_writeback_task+0x63/0x1b0 [<ffffffff8109eaa7>] ?
bit_waitqueue+0x17/0xd0 [<ffffffff81148620>] ? bdi_start_fn+0x0/0x100
[<ffffffff811486a6>] bdi_start_fn+0x86/0x100 [<ffffffff81148620>] ?
bdi_start_fn+0x0/0x100 [<ffffffff8109e78e>] kthread+0x9e/0xc0
[<ffffffff8100c28a>] child_rip+0xa/0x20 [<ffffffff8109e6f0>] ?
kthread+0x0/0xc0 [<ffffffff8100c280>] ? child_rip+0x0/0x20 INFO: task
jbd2/dm-0-8:373 blocked for more than 120 seconds. Not tainted
2.6.32-504.30.3.el6.x86_64 #1 "echo 0 >
/proc/sys/kernel/hung_task_timeout_secs" disables this message.
jbd2/dm-0-8 D 0000000000000000 0 373 2 0x00000000 ffff880037f5bd20
0000000000000046 00000000000158c0 00000000000158c0 ffff88000220f980
00000000000158c0 00000000000158c0 ffff880037e48040 ffffffff81a8d020
ffffffff8160dd00 ffff880037e485f8 ffff880037f5bfd8 Call Trace:
[<ffffffff8109ef4e>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa007480f>]
jbd2_journal_commit_transaction+0x19f/0x1500 [jbd2] [<ffffffff8108742c>]
? lock_timer_base+0x3c/0x70 [<ffffffff8109ec20>] ?
autoremove_wake_function+0x0/0x40 [<ffffffffa007aa48>]
kjournald2+0xb8/0x220 [jbd2] [<ffffffff8109ec20>] ?
autoremove_wake_function+0x0/0x40 [<ffffffffa007a990>] ?
kjournald2+0x0/0x220 [jbd2] [<ffffffff8109e78e>] kthread+0x9e/0xc0
[<ffffffff8100c28a>] child_rip+0xa/0x20 [<ffffffff8109e6f0>] ?
kthread+0x0/0xc0 [<ffffffff8100c280>] ? child_rip+0x0/0x20 INFO: task
auditd:978 blocked for more than 120 seconds. Not tainted
2.6.32-504.30.3.el6.x86_64 #1 "echo 0 >
/proc/sys/kernel/hung_task_timeout_secs" disables this message. auditd D
0000000000000000 0 978 1 0x00000000 ffff88007c261a88 0000000000000086
0000000000000000 ffff88002d1993d8 ffff88002d1993d8 ffff880079cbf4e8
00035e86aaf88734 0000000000000000 ffff88007c261a28 0000000138823544
ffff8800379365f8 ffff88007c261fd8 Call Trace: [<ffffffffa007308a>]
start_this_handle+0x25a/0x480 [jbd2] [<ffffffff8109ec20>] ?
autoremove_wake_function+0x0/0x40 [<ffffffffa0073495>]
jbd2_journal_start+0xb5/0x100 [jbd2] [<ffffffffa00b2e66>]
ext4_journal_start_sb+0x56/0xe0 [ext4] [<ffffffffa009bf0a>]
ext4_dirty_inode+0x2a/0x60 [ext4] [<ffffffff811baa9b>]
__mark_inode_dirty+0x3b/0x160 [<ffffffff811ab182>]
file_update_time+0xf2/0x170 [<ffffffff811268a0>]
__generic_file_aio_write+0x230/0x490 [<ffffffff81126b88>]
generic_file_aio_write+0x88/0x100 [<ffffffffa0095e08>]
ext4_file_write+0x58/0x190 [ext4] [<ffffffff8118e0aa>]
do_sync_write+0xfa/0x140 [<ffffffffa007a5d5>] ?
jbd2_log_wait_commit+0xf5/0x140 [jbd2] [<ffffffffa00b004f>] ?
ext4_statfs+0xef/0x200 [ext4] [<ffffffff8109ec20>] ?
autoremove_wake_function+0x0/0x40 [<ffffffff811c2f38>] ?
do_statfs_native+0x98/0xb0 [<ffffffff8122da86>] ?
security_file_permission+0x16/0x20 [<ffffffff8118e3a8>]
vfs_write+0xb8/0x1a0 [<ffffffff8118ed71>] sys_write+0x51/0x90
[<ffffffff8100b0d2>] system_call_fastpath+0x16/0x1b INFO: task rs:main
Q:Reg:1017 blocked for more than 120 seconds. Not tainted
2.6.32-504.30.3.el6.x86_64 #1 "echo 0 >
/proc/sys/kernel/hung_task_timeout_secs" disables this message. rs:main
Q:Reg D 0000000000000000 0 1017 1 0x00000080 ffff88007a5f3a88
0000000000000086 0000000000000000 00000002000093d8 0000000000000000
ffff88000220f660 00035e86aa672ebb 0000000000000008 00000000000158c0
0000000138823478 ffff880037d01068 ffff88007a5f3fd8 Call Trace:
[<ffffffffa007308a>] start_this_handle+0x25a/0x480 [jbd2]
[<ffffffff81173fbb>] ? cache_alloc_refill+0x15b/0x240
[<ffffffff8109ec20>] ? autoremove_wake_function+0x0/0x40
[<ffffffffa0073495>] jbd2_journal_start+0xb5/0x100 [jbd2]
[<ffffffffa00b2e66>] ext4_journal_start_sb+0x56/0xe0 [ext4]
[<ffffffffa009bf0a>] ext4_dirty_inode+0x2a/0x60 [ext4]
[<ffffffff811baa9b>] __mark_inode_dirty+0x3b/0x160 [<ffffffff811ab182>]
file_update_time+0xf2/0x170 [<ffffffff811268a0>]
__generic_file_aio_write+0x230/0x490 [<ffffffff81126b88>]
generic_file_aio_write+0x88/0x100 [<ffffffffa0095e08>]
ext4_file_write+0x58/0x190 [ext4] [<ffffffff8118e0aa>]
do_sync_write+0xfa/0x140 [<ffffffff8109ec20>] ?
autoremove_wake_function+0x0/0x40 [<ffffffff8122da86>] ?
security_file_permission+0x16/0x20 [<ffffffff8118e3a8>]
vfs_write+0xb8/0x1a0 [<ffffffff8118ed71>] sys_write+0x51/0x90
[<ffffffff810e5c5e>] ? __audit_syscall_exit+0x25e/0x290
[<ffffffff8100b0d2>] system_call_fastpath+0x16/0x1b INFO: task
flush-253:0:1016 blocked for more than 120 seconds. Not tainted
2.6.32-504.30.3.el6.x86_64 #1 "echo 0 >
/proc/sys/kernel/hung_task_timeout_secs" disables this message.
flush-253:0 D 0000000000000000 0 1016 2 0x00000080 ffff880037fbb430
0000000000000046 0000000000000000 ffffffff81041e98 ffff8800ffffffff
00000000092625a3 00035e80853b5e12 ffff880037dcb910 000000000b5a83fa
000000013881ce2a ffff880037b19ad8 ffff880037fbbfd8 Call Trace:
[<ffffffff81041e98>] ? pvclock_clocksource_read+0x58/0xd0
[<ffffffff811c5310>] ? sync_buffer+0x0/0x50 [<ffffffff8152a2f3>]
io_schedule+0x73/0xc0 [<ffffffff811c5350>] sync_buffer+0x40/0x50
[<ffffffff8152ab8a>] __wait_on_bit_lock+0x5a/0xc0 [<ffffffff811c5310>] ?
sync_buffer+0x0/0x50 [<ffffffff8152ac68>]
out_of_line_wait_on_bit_lock+0x78/0x90 [<ffffffff8109eca0>] ?
wake_bit_function+0x0/0x50 [<ffffffff811c48a9>] ?
__find_get_block+0xa9/0x200 [<ffffffff811c54f6>] __lock_buffer+0x36/0x40
[<ffffffffa0074293>] do_get_write_access+0x493/0x520 [jbd2]
[<ffffffffa0074471>] jbd2_journal_get_write_access+0x31/0x50 [jbd2]
[<ffffffffa00c1da8>] __ext4_journal_get_write_access+0x38/0x80 [ext4]
[<ffffffffa009bbb3>] ext4_reserve_inode_write+0x73/0xa0 [ext4]
[<ffffffffa007378f>] ? jbd2_journal_dirty_metadata+0xff/0x150 [jbd2]
[<ffffffffa009bc2c>] ext4_mark_inode_dirty+0x4c/0x1d0 [ext4]
[<ffffffffa009bf20>] ext4_dirty_inode+0x40/0x60 [ext4]
[<ffffffff811baa9b>] __mark_inode_dirty+0x3b/0x160 [<ffffffffa009b241>]
ext4_da_update_reserve_space+0x111/0x2a0 [ext4] [<ffffffffa00bf97d>]
ext4_ext_get_blocks+0x72d/0x14d0 [ext4] [<ffffffff81270c90>] ?
generic_make_request+0x240/0x5a0 [<ffffffff81126c85>] ?
mempool_alloc_slab+0x15/0x20 [<ffffffffa009cbd9>]
ext4_get_blocks+0xf9/0x2b0 [ext4] [<ffffffff8113a7e5>] ?
pagevec_lookup_tag+0x25/0x40 [<ffffffffa00a1211>]
mpage_da_map_and_submit+0xa1/0x470 [ext4] [<ffffffffa0073495>] ?
jbd2_journal_start+0xb5/0x100 [jbd2] [<ffffffffa00a1e2e>]
ext4_da_writepages+0x2ee/0x620 [ext4] [<ffffffff81139871>]
do_writepages+0x21/0x40 [<ffffffff811bb19d>]
writeback_single_inode+0xdd/0x290 [<ffffffff811bb59d>]
writeback_sb_inodes+0xbd/0x170 [<ffffffff811bb6fb>]
writeback_inodes_wb+0xab/0x1b0 [<ffffffff811bbaf3>]
wb_writeback+0x2f3/0x410 [<ffffffff810880b2>] ? del_timer_sync+0x22/0x30
[<ffffffff811bbdb5>] wb_do_writeback+0x1a5/0x240 [<ffffffff811bbeb3>]
bdi_writeback_task+0x63/0x1b0 [<ffffffff8109eaa7>] ?
bit_waitqueue+0x17/0xd0 [<ffffffff81148620>] ? bdi_start_fn+0x0/0x100
[<ffffffff811486a6>] bdi_start_fn+0x86/0x100 [<ffffffff81148620>] ?
bdi_start_fn+0x0/0x100 [<ffffffff8109e78e>] kthread+0x9e/0xc0
[<ffffffff8100c28a>] child_rip+0xa/0x20 [<ffffffff8109e6f0>] ?
kthread+0x0/0xc0 [<ffffffff8100c280>] ? child_rip+0x0/0x20

we use below env made the vm base imges:
qemu-img-1.5.3-86.el7_1.1.x86_64
ipxe-roms-qemu-20130517-6.gitc4bce43.el7.noarch
libvirt-daemon-driver-qemu-1.2.8-16.el7_1.3.x86_64
qemu-kvm-1.5.3-86.el7_1.1.x86_64
qemu-kvm-common-1.5.3-86.el7_1.1.x86_64

but we use below env to run the vm:
ipxe-roms-qemu-20130517-6.gitc4bce43.el7.noarch
qemu-kvm-ev-2.1.2-23.el7_1.8.1.x86_64
qemu-kvm-common-ev-2.1.2-23.el7_1.8.1.x86_64
qemu-img-ev-2.1.2-23.el7_1.8.1.x86_64
libvirt-daemon-driver-qemu-1.2.8-16.el7_1.3.x86_64

host os :centos 7 3.10.0-229.11.1.el7.x86_64
guest os:centos 6 2.6.32-504.30.3.el6.x86_64

can we do that?
any suggestions would be appreciated, thks!

** Affects: nova
     Importance: Undecided
         Status: New

-- 
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to OpenStack Compute (nova).
https://bugs.launchpad.net/bugs/1507620

Title:
  vm i/o blocked random at low i/o Pressure

Status in OpenStack Compute (nova):
  New

Bug description:
   vm occur i/o blocked about every five days. vm log as below:

  INFO: task jbd2/dm-0-8:373 blocked for more than 120 seconds. Not
  tainted 2.6.32-504.30.3.el6.x86_64 #1 "echo 0 >
  /proc/sys/kernel/hung_task_timeout_secs" disables this message.
  jbd2/dm-0-8 D 0000000000000000 0 373 2 0x00000000 ffff880037f5bd20
  0000000000000046 00000000000158c0 00000000000158c0 ffff88000220f980
  00000000000158c0 00000000000158c0 ffff880037e48040 ffffffff81a8d020
  ffffffff8160dd00 ffff880037e485f8 ffff880037f5bfd8 Call Trace:
  [<ffffffff8109ef4e>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa007480f>]
  jbd2_journal_commit_transaction+0x19f/0x1500 [jbd2]
  [<ffffffff8108742c>] ? lock_timer_base+0x3c/0x70 [<ffffffff8109ec20>]
  ? autoremove_wake_function+0x0/0x40 [<ffffffffa007aa48>]
  kjournald2+0xb8/0x220 [jbd2] [<ffffffff8109ec20>] ?
  autoremove_wake_function+0x0/0x40 [<ffffffffa007a990>] ?
  kjournald2+0x0/0x220 [jbd2] [<ffffffff8109e78e>] kthread+0x9e/0xc0
  [<ffffffff8100c28a>] child_rip+0xa/0x20 [<ffffffff8109e6f0>] ?
  kthread+0x0/0xc0 [<ffffffff8100c280>] ? child_rip+0x0/0x20 INFO: task
  auditd:978 blocked for more than 120 seconds. Not tainted
  2.6.32-504.30.3.el6.x86_64 #1 "echo 0 >
  /proc/sys/kernel/hung_task_timeout_secs" disables this message. auditd
  D 0000000000000000 0 978 1 0x00000000 ffff88007c261a88
  0000000000000086 0000000000000000 ffff88002d1993d8 ffff88002d1993d8
  ffff880079cbf4e8 00035e86aaf88734 0000000000000000 ffff88007c261a28
  0000000138823544 ffff8800379365f8 ffff88007c261fd8 Call Trace:
  [<ffffffffa007308a>] start_this_handle+0x25a/0x480 [jbd2]
  [<ffffffff8109ec20>] ? autoremove_wake_function+0x0/0x40
  [<ffffffffa0073495>] jbd2_journal_start+0xb5/0x100 [jbd2]
  [<ffffffffa00b2e66>] ext4_journal_start_sb+0x56/0xe0 [ext4]
  [<ffffffffa009bf0a>] ext4_dirty_inode+0x2a/0x60 [ext4]
  [<ffffffff811baa9b>] __mark_inode_dirty+0x3b/0x160
  [<ffffffff811ab182>] file_update_time+0xf2/0x170 [<ffffffff811268a0>]
  __generic_file_aio_write+0x230/0x490 [<ffffffff81126b88>]
  generic_file_aio_write+0x88/0x100 [<ffffffffa0095e08>]
  ext4_file_write+0x58/0x190 [ext4] [<ffffffff8118e0aa>]
  do_sync_write+0xfa/0x140 [<ffffffffa007a5d5>] ?
  jbd2_log_wait_commit+0xf5/0x140 [jbd2] [<ffffffffa00b004f>] ?
  ext4_statfs+0xef/0x200 [ext4] [<ffffffff8109ec20>] ?
  autoremove_wake_function+0x0/0x40 [<ffffffff811c2f38>] ?
  do_statfs_native+0x98/0xb0 [<ffffffff8122da86>] ?
  security_file_permission+0x16/0x20 [<ffffffff8118e3a8>]
  vfs_write+0xb8/0x1a0 [<ffffffff8118ed71>] sys_write+0x51/0x90
  [<ffffffff8100b0d2>] system_call_fastpath+0x16/0x1b INFO: task rs:main
  Q:Reg:1017 blocked for more than 120 seconds. Not tainted
  2.6.32-504.30.3.el6.x86_64 #1 "echo 0 >
  /proc/sys/kernel/hung_task_timeout_secs" disables this message.
  rs:main Q:Reg D 0000000000000000 0 1017 1 0x00000080 ffff88007a5f3a88
  0000000000000086 0000000000000000 00000002000093d8 0000000000000000
  ffff88000220f660 00035e86aa672ebb 0000000000000008 00000000000158c0
  0000000138823478 ffff880037d01068 ffff88007a5f3fd8 Call Trace:
  [<ffffffffa007308a>] start_this_handle+0x25a/0x480 [jbd2]
  [<ffffffff81173fbb>] ? cache_alloc_refill+0x15b/0x240
  [<ffffffff8109ec20>] ? autoremove_wake_function+0x0/0x40
  [<ffffffffa0073495>] jbd2_journal_start+0xb5/0x100 [jbd2]
  [<ffffffffa00b2e66>] ext4_journal_start_sb+0x56/0xe0 [ext4]
  [<ffffffffa009bf0a>] ext4_dirty_inode+0x2a/0x60 [ext4]
  [<ffffffff811baa9b>] __mark_inode_dirty+0x3b/0x160
  [<ffffffff811ab182>] file_update_time+0xf2/0x170 [<ffffffff811268a0>]
  __generic_file_aio_write+0x230/0x490 [<ffffffff81126b88>]
  generic_file_aio_write+0x88/0x100 [<ffffffffa0095e08>]
  ext4_file_write+0x58/0x190 [ext4] [<ffffffff8118e0aa>]
  do_sync_write+0xfa/0x140 [<ffffffff8109ec20>] ?
  autoremove_wake_function+0x0/0x40 [<ffffffff8122da86>] ?
  security_file_permission+0x16/0x20 [<ffffffff8118e3a8>]
  vfs_write+0xb8/0x1a0 [<ffffffff8118ed71>] sys_write+0x51/0x90
  [<ffffffff810e5c5e>] ? __audit_syscall_exit+0x25e/0x290
  [<ffffffff8100b0d2>] system_call_fastpath+0x16/0x1b INFO: task
  flush-253:0:1016 blocked for more than 120 seconds. Not tainted
  2.6.32-504.30.3.el6.x86_64 #1 "echo 0 >
  /proc/sys/kernel/hung_task_timeout_secs" disables this message.
  flush-253:0 D 0000000000000000 0 1016 2 0x00000080 ffff880037fbb430
  0000000000000046 0000000000000000 ffffffff81041e98 ffff8800ffffffff
  00000000092625a3 00035e80853b5e12 ffff880037dcb910 000000000b5a83fa
  000000013881ce2a ffff880037b19ad8 ffff880037fbbfd8 Call Trace:
  [<ffffffff81041e98>] ? pvclock_clocksource_read+0x58/0xd0
  [<ffffffff811c5310>] ? sync_buffer+0x0/0x50 [<ffffffff8152a2f3>]
  io_schedule+0x73/0xc0 [<ffffffff811c5350>] sync_buffer+0x40/0x50
  [<ffffffff8152ab8a>] __wait_on_bit_lock+0x5a/0xc0 [<ffffffff811c5310>]
  ? sync_buffer+0x0/0x50 [<ffffffff8152ac68>]
  out_of_line_wait_on_bit_lock+0x78/0x90 [<ffffffff8109eca0>] ?
  wake_bit_function+0x0/0x50 [<ffffffff811c48a9>] ?
  __find_get_block+0xa9/0x200 [<ffffffff811c54f6>]
  __lock_buffer+0x36/0x40 [<ffffffffa0074293>]
  do_get_write_access+0x493/0x520 [jbd2] [<ffffffffa0074471>]
  jbd2_journal_get_write_access+0x31/0x50 [jbd2] [<ffffffffa00c1da8>]
  __ext4_journal_get_write_access+0x38/0x80 [ext4] [<ffffffffa009bbb3>]
  ext4_reserve_inode_write+0x73/0xa0 [ext4] [<ffffffffa007378f>] ?
  jbd2_journal_dirty_metadata+0xff/0x150 [jbd2] [<ffffffffa009bc2c>]
  ext4_mark_inode_dirty+0x4c/0x1d0 [ext4] [<ffffffffa009bf20>]
  ext4_dirty_inode+0x40/0x60 [ext4] [<ffffffff811baa9b>]
  __mark_inode_dirty+0x3b/0x160 [<ffffffffa009b241>]
  ext4_da_update_reserve_space+0x111/0x2a0 [ext4] [<ffffffffa00bf97d>]
  ext4_ext_get_blocks+0x72d/0x14d0 [ext4] [<ffffffff81270c90>] ?
  generic_make_request+0x240/0x5a0 [<ffffffff81126c85>] ?
  mempool_alloc_slab+0x15/0x20 [<ffffffffa009cbd9>]
  ext4_get_blocks+0xf9/0x2b0 [ext4] [<ffffffff8113a7e5>] ?
  pagevec_lookup_tag+0x25/0x40 [<ffffffffa00a1211>]
  mpage_da_map_and_submit+0xa1/0x470 [ext4] [<ffffffffa0073495>] ?
  jbd2_journal_start+0xb5/0x100 [jbd2] [<ffffffffa00a1e2e>]
  ext4_da_writepages+0x2ee/0x620 [ext4] [<ffffffff81139871>]
  do_writepages+0x21/0x40 [<ffffffff811bb19d>]
  writeback_single_inode+0xdd/0x290 [<ffffffff811bb59d>]
  writeback_sb_inodes+0xbd/0x170 [<ffffffff811bb6fb>]
  writeback_inodes_wb+0xab/0x1b0 [<ffffffff811bbaf3>]
  wb_writeback+0x2f3/0x410 [<ffffffff810880b2>] ?
  del_timer_sync+0x22/0x30 [<ffffffff811bbdb5>]
  wb_do_writeback+0x1a5/0x240 [<ffffffff811bbeb3>]
  bdi_writeback_task+0x63/0x1b0 [<ffffffff8109eaa7>] ?
  bit_waitqueue+0x17/0xd0 [<ffffffff81148620>] ? bdi_start_fn+0x0/0x100
  [<ffffffff811486a6>] bdi_start_fn+0x86/0x100 [<ffffffff81148620>] ?
  bdi_start_fn+0x0/0x100 [<ffffffff8109e78e>] kthread+0x9e/0xc0
  [<ffffffff8100c28a>] child_rip+0xa/0x20 [<ffffffff8109e6f0>] ?
  kthread+0x0/0xc0 [<ffffffff8100c280>] ? child_rip+0x0/0x20 INFO: task
  jbd2/dm-0-8:373 blocked for more than 120 seconds. Not tainted
  2.6.32-504.30.3.el6.x86_64 #1 "echo 0 >
  /proc/sys/kernel/hung_task_timeout_secs" disables this message.
  jbd2/dm-0-8 D 0000000000000000 0 373 2 0x00000000 ffff880037f5bd20
  0000000000000046 00000000000158c0 00000000000158c0 ffff88000220f980
  00000000000158c0 00000000000158c0 ffff880037e48040 ffffffff81a8d020
  ffffffff8160dd00 ffff880037e485f8 ffff880037f5bfd8 Call Trace:
  [<ffffffff8109ef4e>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa007480f>]
  jbd2_journal_commit_transaction+0x19f/0x1500 [jbd2]
  [<ffffffff8108742c>] ? lock_timer_base+0x3c/0x70 [<ffffffff8109ec20>]
  ? autoremove_wake_function+0x0/0x40 [<ffffffffa007aa48>]
  kjournald2+0xb8/0x220 [jbd2] [<ffffffff8109ec20>] ?
  autoremove_wake_function+0x0/0x40 [<ffffffffa007a990>] ?
  kjournald2+0x0/0x220 [jbd2] [<ffffffff8109e78e>] kthread+0x9e/0xc0
  [<ffffffff8100c28a>] child_rip+0xa/0x20 [<ffffffff8109e6f0>] ?
  kthread+0x0/0xc0 [<ffffffff8100c280>] ? child_rip+0x0/0x20 INFO: task
  auditd:978 blocked for more than 120 seconds. Not tainted
  2.6.32-504.30.3.el6.x86_64 #1 "echo 0 >
  /proc/sys/kernel/hung_task_timeout_secs" disables this message. auditd
  D 0000000000000000 0 978 1 0x00000000 ffff88007c261a88
  0000000000000086 0000000000000000 ffff88002d1993d8 ffff88002d1993d8
  ffff880079cbf4e8 00035e86aaf88734 0000000000000000 ffff88007c261a28
  0000000138823544 ffff8800379365f8 ffff88007c261fd8 Call Trace:
  [<ffffffffa007308a>] start_this_handle+0x25a/0x480 [jbd2]
  [<ffffffff8109ec20>] ? autoremove_wake_function+0x0/0x40
  [<ffffffffa0073495>] jbd2_journal_start+0xb5/0x100 [jbd2]
  [<ffffffffa00b2e66>] ext4_journal_start_sb+0x56/0xe0 [ext4]
  [<ffffffffa009bf0a>] ext4_dirty_inode+0x2a/0x60 [ext4]
  [<ffffffff811baa9b>] __mark_inode_dirty+0x3b/0x160
  [<ffffffff811ab182>] file_update_time+0xf2/0x170 [<ffffffff811268a0>]
  __generic_file_aio_write+0x230/0x490 [<ffffffff81126b88>]
  generic_file_aio_write+0x88/0x100 [<ffffffffa0095e08>]
  ext4_file_write+0x58/0x190 [ext4] [<ffffffff8118e0aa>]
  do_sync_write+0xfa/0x140 [<ffffffffa007a5d5>] ?
  jbd2_log_wait_commit+0xf5/0x140 [jbd2] [<ffffffffa00b004f>] ?
  ext4_statfs+0xef/0x200 [ext4] [<ffffffff8109ec20>] ?
  autoremove_wake_function+0x0/0x40 [<ffffffff811c2f38>] ?
  do_statfs_native+0x98/0xb0 [<ffffffff8122da86>] ?
  security_file_permission+0x16/0x20 [<ffffffff8118e3a8>]
  vfs_write+0xb8/0x1a0 [<ffffffff8118ed71>] sys_write+0x51/0x90
  [<ffffffff8100b0d2>] system_call_fastpath+0x16/0x1b INFO: task rs:main
  Q:Reg:1017 blocked for more than 120 seconds. Not tainted
  2.6.32-504.30.3.el6.x86_64 #1 "echo 0 >
  /proc/sys/kernel/hung_task_timeout_secs" disables this message.
  rs:main Q:Reg D 0000000000000000 0 1017 1 0x00000080 ffff88007a5f3a88
  0000000000000086 0000000000000000 00000002000093d8 0000000000000000
  ffff88000220f660 00035e86aa672ebb 0000000000000008 00000000000158c0
  0000000138823478 ffff880037d01068 ffff88007a5f3fd8 Call Trace:
  [<ffffffffa007308a>] start_this_handle+0x25a/0x480 [jbd2]
  [<ffffffff81173fbb>] ? cache_alloc_refill+0x15b/0x240
  [<ffffffff8109ec20>] ? autoremove_wake_function+0x0/0x40
  [<ffffffffa0073495>] jbd2_journal_start+0xb5/0x100 [jbd2]
  [<ffffffffa00b2e66>] ext4_journal_start_sb+0x56/0xe0 [ext4]
  [<ffffffffa009bf0a>] ext4_dirty_inode+0x2a/0x60 [ext4]
  [<ffffffff811baa9b>] __mark_inode_dirty+0x3b/0x160
  [<ffffffff811ab182>] file_update_time+0xf2/0x170 [<ffffffff811268a0>]
  __generic_file_aio_write+0x230/0x490 [<ffffffff81126b88>]
  generic_file_aio_write+0x88/0x100 [<ffffffffa0095e08>]
  ext4_file_write+0x58/0x190 [ext4] [<ffffffff8118e0aa>]
  do_sync_write+0xfa/0x140 [<ffffffff8109ec20>] ?
  autoremove_wake_function+0x0/0x40 [<ffffffff8122da86>] ?
  security_file_permission+0x16/0x20 [<ffffffff8118e3a8>]
  vfs_write+0xb8/0x1a0 [<ffffffff8118ed71>] sys_write+0x51/0x90
  [<ffffffff810e5c5e>] ? __audit_syscall_exit+0x25e/0x290
  [<ffffffff8100b0d2>] system_call_fastpath+0x16/0x1b INFO: task
  flush-253:0:1016 blocked for more than 120 seconds. Not tainted
  2.6.32-504.30.3.el6.x86_64 #1 "echo 0 >
  /proc/sys/kernel/hung_task_timeout_secs" disables this message.
  flush-253:0 D 0000000000000000 0 1016 2 0x00000080 ffff880037fbb430
  0000000000000046 0000000000000000 ffffffff81041e98 ffff8800ffffffff
  00000000092625a3 00035e80853b5e12 ffff880037dcb910 000000000b5a83fa
  000000013881ce2a ffff880037b19ad8 ffff880037fbbfd8 Call Trace:
  [<ffffffff81041e98>] ? pvclock_clocksource_read+0x58/0xd0
  [<ffffffff811c5310>] ? sync_buffer+0x0/0x50 [<ffffffff8152a2f3>]
  io_schedule+0x73/0xc0 [<ffffffff811c5350>] sync_buffer+0x40/0x50
  [<ffffffff8152ab8a>] __wait_on_bit_lock+0x5a/0xc0 [<ffffffff811c5310>]
  ? sync_buffer+0x0/0x50 [<ffffffff8152ac68>]
  out_of_line_wait_on_bit_lock+0x78/0x90 [<ffffffff8109eca0>] ?
  wake_bit_function+0x0/0x50 [<ffffffff811c48a9>] ?
  __find_get_block+0xa9/0x200 [<ffffffff811c54f6>]
  __lock_buffer+0x36/0x40 [<ffffffffa0074293>]
  do_get_write_access+0x493/0x520 [jbd2] [<ffffffffa0074471>]
  jbd2_journal_get_write_access+0x31/0x50 [jbd2] [<ffffffffa00c1da8>]
  __ext4_journal_get_write_access+0x38/0x80 [ext4] [<ffffffffa009bbb3>]
  ext4_reserve_inode_write+0x73/0xa0 [ext4] [<ffffffffa007378f>] ?
  jbd2_journal_dirty_metadata+0xff/0x150 [jbd2] [<ffffffffa009bc2c>]
  ext4_mark_inode_dirty+0x4c/0x1d0 [ext4] [<ffffffffa009bf20>]
  ext4_dirty_inode+0x40/0x60 [ext4] [<ffffffff811baa9b>]
  __mark_inode_dirty+0x3b/0x160 [<ffffffffa009b241>]
  ext4_da_update_reserve_space+0x111/0x2a0 [ext4] [<ffffffffa00bf97d>]
  ext4_ext_get_blocks+0x72d/0x14d0 [ext4] [<ffffffff81270c90>] ?
  generic_make_request+0x240/0x5a0 [<ffffffff81126c85>] ?
  mempool_alloc_slab+0x15/0x20 [<ffffffffa009cbd9>]
  ext4_get_blocks+0xf9/0x2b0 [ext4] [<ffffffff8113a7e5>] ?
  pagevec_lookup_tag+0x25/0x40 [<ffffffffa00a1211>]
  mpage_da_map_and_submit+0xa1/0x470 [ext4] [<ffffffffa0073495>] ?
  jbd2_journal_start+0xb5/0x100 [jbd2] [<ffffffffa00a1e2e>]
  ext4_da_writepages+0x2ee/0x620 [ext4] [<ffffffff81139871>]
  do_writepages+0x21/0x40 [<ffffffff811bb19d>]
  writeback_single_inode+0xdd/0x290 [<ffffffff811bb59d>]
  writeback_sb_inodes+0xbd/0x170 [<ffffffff811bb6fb>]
  writeback_inodes_wb+0xab/0x1b0 [<ffffffff811bbaf3>]
  wb_writeback+0x2f3/0x410 [<ffffffff810880b2>] ?
  del_timer_sync+0x22/0x30 [<ffffffff811bbdb5>]
  wb_do_writeback+0x1a5/0x240 [<ffffffff811bbeb3>]
  bdi_writeback_task+0x63/0x1b0 [<ffffffff8109eaa7>] ?
  bit_waitqueue+0x17/0xd0 [<ffffffff81148620>] ? bdi_start_fn+0x0/0x100
  [<ffffffff811486a6>] bdi_start_fn+0x86/0x100 [<ffffffff81148620>] ?
  bdi_start_fn+0x0/0x100 [<ffffffff8109e78e>] kthread+0x9e/0xc0
  [<ffffffff8100c28a>] child_rip+0xa/0x20 [<ffffffff8109e6f0>] ?
  kthread+0x0/0xc0 [<ffffffff8100c280>] ? child_rip+0x0/0x20

  we use below env made the vm base imges:
  qemu-img-1.5.3-86.el7_1.1.x86_64
  ipxe-roms-qemu-20130517-6.gitc4bce43.el7.noarch
  libvirt-daemon-driver-qemu-1.2.8-16.el7_1.3.x86_64
  qemu-kvm-1.5.3-86.el7_1.1.x86_64
  qemu-kvm-common-1.5.3-86.el7_1.1.x86_64

  but we use below env to run the vm:
  ipxe-roms-qemu-20130517-6.gitc4bce43.el7.noarch
  qemu-kvm-ev-2.1.2-23.el7_1.8.1.x86_64
  qemu-kvm-common-ev-2.1.2-23.el7_1.8.1.x86_64
  qemu-img-ev-2.1.2-23.el7_1.8.1.x86_64
  libvirt-daemon-driver-qemu-1.2.8-16.el7_1.3.x86_64

  host os :centos 7 3.10.0-229.11.1.el7.x86_64
  guest os:centos 6 2.6.32-504.30.3.el6.x86_64

  can we do that?
  any suggestions would be appreciated, thks!

To manage notifications about this bug go to:
https://bugs.launchpad.net/nova/+bug/1507620/+subscriptions