← Back to team overview

kernel-packages team mailing list archive

  1. kernel-packages team
  2. Mailing list archive
  3. Message #131541

[Bug 1486502] [NEW] Ubuntu15.10: KVM guest got hung (console) while running stress tool (stress tool with cpuhotplug)

  Thread Previous • Date Previous • • Thread Next 
You have been subscribed to a public bug:

Problem Description
====================
  I installed Ubuntu15.10 having kernel 4.1.0-1-generic (kernel) on top of PowerKVM 3.1 host , I assigned 2 cpu to KVM guest and schedule  stress test (cpu) for more than 10 hour.

After ran the test  it having call traces related to
hung_task_timeout_secs And  system got hung while running the tests .


TEST  was ran : 

1-stress tool for 10 h stress --cpu 8 --io 4 --vm 2 --vm-bytes 128M --timeout 10h
2- cpu hotplug test (cpustress.sh)

 root@ubuntu:~/cpu_test# cat cpustress.sh
#!/bin/bash
var=$1
END=$10000000
for ((j=1;j<=$END;j++));
do
    for ((i=0;i<=var;i++));
            do
                    echo $i
            test=`cat /sys/devices/system/cpu/cpu$i/online`
            if [ $test == 0 ]
            then
                echo 1 > /sys/devices/system/cpu/cpu$i/online
            else
                echo 0 > /sys/devices/system/cpu/cpu$i/online
            fi
            sleep 2
                `cat /proc/cpuinfo > cpulog.text`
        
            done
done
root@ubuntu:~/cpu_test#

ran parallel  both test .


LOG:

root@ubuntu:~/cpu_test# 
[root@powerkvm5-lp1 ~]# 

root@ubuntu:~# cat /proc/cpuinfo 
processor	: 0
cpu		: POWER8E (raw), altivec supported
clock		: 4157.000000MHz
revision	: 2.1 (pvr 004b 0201)

processor	: 2
cpu		: POWER8E (raw), altivec supported
clock		: 4157.000000MHz
revision	: 2.1 (pvr 004b 0201)

timebase	: 512000000
platform	: pSeries
model		: IBM pSeries (emulated by qemu)
machine		: CHRP IBM pSeries (emulated by qemu)
root@ubuntu:~# 


cat /var/log/syslog

[ 4440.010656] [c00000007700fe30] [c000000000009560] ret_from_kernel_thread+0x5c/0x7c
[ 4440.010758] INFO: task stress:670 blocked for more than 120 seconds.
[ 4440.010795]       Tainted: G        W    L  4.1.0-1-generic #7-Ubuntu
[ 4440.010831] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 4440.010874] stress          D 00003fffa367d488     0   670    668 0x00040000
[ 4440.010876] Call Trace:
[ 4440.010877] [c00000007514f800] [c00000007514f850] 0xc00000007514f850 (unreliable)
[ 4440.010879] [c00000007514f9d0] [c0000000000157cc] __switch_to+0x1fc/0x350
[ 4440.010880] [c00000007514fa20] [c000000000a3f2a0] __schedule+0x350/0x8a0
[ 4440.010882] [c00000007514faa0] [c000000000a3f834] schedule+0x44/0xc0
[ 4440.010883] [c00000007514fad0] [c000000000a438d4] schedule_timeout+0x254/0x2f0
[ 4440.010885] [c00000007514fbc0] [c000000000a4085c] wait_for_common+0xec/0x240
[ 4440.010892] [c00000007514fc40] [c0000000004d4cc4] submit_bio_wait+0x84/0xb0
[ 4440.010894] [c00000007514fcb0] [c0000000004e6e08] blkdev_issue_flush+0x88/0xe0
[ 4440.010896] [c00000007514fcf0] [c000000000394e0c] ext4_sync_fs+0x1ac/0x280
[ 4440.010898] [c00000007514fd50] [c0000000002fc1c0] sync_fs_one_sb+0x60/0x80
[ 4440.010900] [c00000007514fd80] [c0000000002ba768] iterate_supers+0x1b8/0x200
[ 4440.010902] [c00000007514fdf0] [c0000000002fc368] sys_sync+0x78/0xf0
[ 4440.010904] [c00000007514fe30] [c000000000009258] system_call+0x38/0xd0
[ 4440.010905] INFO: task stress:673 blocked for more than 120 seconds.
[ 4440.010941]       Tainted: G        W    L  4.1.0-1-generic #7-Ubuntu
[ 4440.010977] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 4440.011020] stress          D 00003fffa367d488     0   673    668 0x00040000
[ 4440.011022] Call Trace:
[ 4440.011024] [c00000007516b800] [c00000007516b890] 0xc00000007516b890 (unreliable)
[ 4440.011025] [c00000007516b9d0] [c0000000000157cc] __switch_to+0x1fc/0x350
[ 4440.011027] [c00000007516ba20] [c000000000a3f2a0] __schedule+0x350/0x8a0
[ 4440.011028] [c00000007516baa0] [c000000000a3f834] schedule+0x44/0xc0
[ 4440.011030] [c00000007516bad0] [c000000000a438d4] schedule_timeout+0x254/0x2f0
[ 4440.011031] [c00000007516bbc0] [c000000000a4085c] wait_for_common+0xec/0x240
[ 4440.011033] [c00000007516bc40] [c0000000004d4cc4] submit_bio_wait+0x84/0xb0
[ 4440.011034] [c00000007516bcb0] [c0000000004e6e08] blkdev_issue_flush+0x88/0xe0
[ 4440.011036] [c00000007516bcf0] [c000000000394e0c] ext4_sync_fs+0x1ac/0x280
[ 4440.011037] [c00000007516bd50] [c0000000002fc1c0] sync_fs_one_sb+0x60/0x80
[ 4440.011039] [c00000007516bd80] [c0000000002ba768] iterate_supers+0x1b8/0x200
[ 4440.011040] [c00000007516bdf0] [c0000000002fc368] sys_sync+0x78/0xf0
[ 4440.011042] [c00000007516be30] [c000000000009258] system_call+0x38/0xd0
[ 4440.011043] INFO: task stress:676 blocked for more than 120 seconds.
[ 4440.011080]       Tainted: G        W    L  4.1.0-1-generic #7-Ubuntu
[ 4440.011115] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 4440.011158] stress          D 00003fffa367d488     0   676    668 0x00040000
[ 4440.011159] Call Trace:
[ 4440.011161] [c000000075177800] [c000000075177890] 0xc000000075177890 (unreliable)
[ 4440.011163] [c0000000751779d0] [c0000000000157cc] __switch_to+0x1fc/0x350
[ 4440.011164] [c000000075177a20] [c000000000a3f2a0] __schedule+0x350/0x8a0
[ 4440.011165] [c000000075177aa0] [c000000000a3f834] schedule+0x44/0xc0
[ 4440.011167] [c000000075177ad0] [c000000000a438d4] schedule_timeout+0x254/0x2f0
[ 4440.011168] [c000000075177bc0] [c000000000a4085c] wait_for_common+0xec/0x240
[ 4440.011170] [c000000075177c40] [c0000000004d4cc4] submit_bio_wait+0x84/0xb0
[ 4440.011171] [c000000075177cb0] [c0000000004e6e08] blkdev_issue_flush+0x88/0xe0
[ 4440.011173] [c000000075177cf0] [c000000000394e0c] ext4_sync_fs+0x1ac/0x280
[ 4440.011174] [c000000075177d50] [c0000000002fc1c0] sync_fs_one_sb+0x60/0x80
[ 4440.011175] [c000000075177d80] [c0000000002ba768] iterate_supers+0x1b8/0x200
[ 4440.011177] [c000000075177df0] [c0000000002fc368] sys_sync+0x78/0xf0
[ 4440.011178] [c000000075177e30] [c000000000009258] system_call+0x38/0xd0
[ 4440.011180] INFO: task stress:678 blocked for more than 120 seconds.
[ 4440.011217]       Tainted: G        W    L  4.1.0-1-generic #7-Ubuntu
[ 4440.011252] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 4440.011295] stress          D 00003fffa367d488     0   678    668 0x00040000
[ 4440.011297] Call Trace:
[ 4440.011299] [c00000007517f990] [c00000007517f9d0] 0xc00000007517f9d0 (unreliable)
[ 4440.011300] [c00000007517fb60] [c0000000000157cc] __switch_to+0x1fc/0x350
[ 4440.011302] [c00000007517fbb0] [c000000000a3f2a0] __schedule+0x350/0x8a0
[ 4440.011303] [c00000007517fc30] [c000000000a3f834] schedule+0x44/0xc0
[ 4440.011305] [c00000007517fc60] [c0000000003d7c74] jbd2_log_wait_commit+0xd4/0x180
[ 4440.011307] [c00000007517fcf0] [c000000000394e4c] ext4_sync_fs+0x1ec/0x280
[ 4440.011308] [c00000007517fd50] [c0000000002fc1c0] sync_fs_one_sb+0x60/0x80
[ 4440.011309] [c00000007517fd80] [c0000000002ba768] iterate_supers+0x1b8/0x200
[ 4440.011311] [c00000007517fdf0] [c0000000002fc368] sys_sync+0x78/0xf0
[ 4440.011313] [c00000007517fe30] [c000000000009258] system_call+0x38/0xd0
[ 4440.011316] INFO: task cpustress.sh:744 blocked for more than 120 seconds.
[ 4440.011352]       Tainted: G        W    L  4.1.0-1-generic #7-Ubuntu
[ 4440.011388] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 4440.011430] cpustress.sh    D 00003fff8b0f4088     0   744    655 0x00040000
[ 4440.011439] Call Trace:
[ 4440.011440] [c0000000758336a0] [c0000000758336e0] 0xc0000000758336e0 (unreliable)
[ 4440.011442] [c000000075833870] [c0000000000157cc] __switch_to+0x1fc/0x350
[ 4440.011443] [c0000000758338c0] [c000000000a3f2a0] __schedule+0x350/0x8a0
[ 4440.011444] [c000000075833940] [c000000000a3f834] schedule+0x44/0xc0
[ 4440.011446] [c000000075833970] [c0000000004f05a4] blk_mq_freeze_queue_wait+0x74/0x110
[ 4440.011448] [c0000000758339e0] [c0000000004f2728] blk_mq_queue_reinit_notify+0x108/0x230
[ 4440.011450] [c000000075833a30] [c0000000000dc698] notifier_call_chain+0x98/0x100
[ 4440.011451] [c000000075833a80] [c0000000000af7c8] cpu_notify+0x48/0xa0
[ 4440.011453] [c000000075833ab0] [c0000000000afc34] _cpu_up+0x214/0x220
[ 4440.011455] [c000000075833b60] [c0000000000afd5c] cpu_up+0x11c/0x140
[ 4440.011462] [c000000075833be0] [c0000000008bb0b4] cpu_subsys_online+0x64/0xf0
[ 4440.011470] [c000000075833c30] [c000000000662c64] device_online+0xb4/0x120
[ 4440.011472] [c000000075833c70] [c000000000662d84] online_store+0xb4/0xc0
[ 4440.011474] [c000000075833cc0] [c00000000065eb38] dev_attr_store+0x68/0xa0
[ 4440.011475] [c000000075833d00] [c000000000361650] sysfs_kf_write+0x80/0xb0
[ 4440.011477] [c000000075833d40] [c000000000360578] kernfs_fop_write+0x188/0x200
[ 4440.011479] [c000000075833d90] [c0000000002b519c] vfs_write+0xdc/0x260
[ 4440.011481] [c000000075833de0] [c0000000002b601c] SyS_write+0x6c/0x110
[ 4440.011482] [c000000075833e30] [c000000000009258] system_call+0x38/0xd0
[ 4560.003242] INFO: task kworker/0:0:4 blocked for more than 120 seconds.
[ 4560.003324]       Tainted: G        W    L  4.1.0-1-generic #7-Ubuntu
[ 4560.003359] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 4560.003402] kworker/0:0     D 0000000000000000     0     4      2 0x00000800
[ 4560.003463] Workqueue: events cpuset_hotplug_workfn
[ 4560.003468] Call Trace:
[ 4560.003482] [c00000007e90f700] [c00000000162a298] fallback_doms+0x0/0x100 (unreliable)
[ 4560.003489] [c00000007e90f8d0] [c0000000000157cc] __switch_to+0x1fc/0x350
[ 4560.003501] [c00000007e90f920] [c000000000a3f2a0] __schedule+0x350/0x8a0
[ 4560.003503] [c00000007e90f9a0] [c000000000a3f834] schedule+0x44/0xc0
[ 4560.003504] [c00000007e90f9d0] [c000000000a3fd90] schedule_preempt_disabled+0x20/0x30
[ 4560.003506] [c00000007e90f9f0] [c000000000a41f5c] __mutex_lock_slowpath+0xec/0x1f0
[ 4560.003507] [c00000007e90fa70] [c000000000a420d8] mutex_lock+0x78/0xa0
[ 4560.003510] [c00000007e90faa0] [c0000000000af668] get_online_cpus+0x58/0xa0
[ 4560.003512] [c00000007e90fad0] [c000000000171d50] rebuild_sched_domains_locked+0x20/0x80
[ 4560.003513] [c00000007e90fb00] [c000000000174af8] rebuild_sched_domains+0x38/0x60
[ 4560.003515] [c00000007e90fb30] [c000000000174ebc] cpuset_hotplug_workfn+0x39c/0x8a0
[ 4560.003517] [c00000007e90fc50] [c0000000000d3084] process_one_work+0x1a4/0x4c0
[ 4560.003519] [c00000007e90fce0] [c0000000000d39a0] worker_thread+0x190/0x600
[ 4560.003520] [c00000007e90fd80] [c0000000000dada0] kthread+0x110/0x130
[ 4560.003522] [c00000007e90fe30] [c000000000009560] ret_from_kernel_thread+0x5c/0x7c
[ 4560.003526] INFO: task kworker/1:0:14 blocked for more than 120 seconds.
[ 4560.003561]       Tainted: G        W    L  4.1.0-1-generic #7-Ubuntu
[ 4560.003596] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 4560.003637] kworker/1:0     D 0000000000000000     0    14      2 0x00000800
[ 4560.003648] Workqueue: events rtas_event_scan
[ 4560.003649] Call Trace:
[ 4560.003657] [c00000007e9537e0] [c0000000013bd818] sysctl_table_root+0x0/0x68 (unreliable)
[ 4560.003659] [c00000007e9539b0] [c0000000000157cc] __switch_to+0x1fc/0x350
[ 4560.003660] [c00000007e953a00] [c000000000a3f2a0] __schedule+0x350/0x8a0
[ 4560.003661] [c00000007e953a80] [c000000000a3f834] schedule+0x44/0xc0
[ 4560.003663] [c00000007e953ab0] [c000000000a3fd90] schedule_preempt_disabled+0x20/0x30
[ 4560.003664] [c00000007e953ad0] [c000000000a41f5c] __mutex_lock_slowpath+0xec/0x1f0
[ 4560.003666] [c00000007e953b50] [c000000000a420d8] mutex_lock+0x78/0xa0
[ 4560.003667] [c00000007e953b80] [c0000000000af668] get_online_cpus+0x58/0xa0
[ 4560.003669] [c00000007e953bb0] [c00000000002da48] rtas_event_scan+0xb8/0x310
[ 4560.003670] [c00000007e953c50] [c0000000000d3084] process_one_work+0x1a4/0x4c0
[ 4560.003672] [c00000007e953ce0] [c0000000000d39a0] worker_thread+0x190/0x600
[ 4560.003673] [c00000007e953d80] [c0000000000dada0] kthread+0x110/0x130
[ 4560.003675] [c00000007e953e30] [c000000000009560] ret_from_kernel_thread+0x5c/0x7c
root@ubuntu:~/cpu_test#


[root@KVMHOST ~]# virsh console PRA_ubuntu1510
Connected to domain PRA_ubuntu1510
Escape character is ^]

root@ubuntu:~# service ssh


[root@KVMHOST ~]# ssh root@192.168.122.53
root@192.168.122.53's password: 
Write failed: Broken pipe
[root@KVMHOST ~]# 


Regards
Praveen

** Affects: linux (Ubuntu)
     Importance: Undecided
         Status: New


** Tags: architecture-ppc64le bot-comment bugnameltc-127970 severity-high targetmilestone-inin---
-- 
Ubuntu15.10: KVM guest  got hung (console) while running stress tool (stress tool with cpuhotplug)
https://bugs.launchpad.net/bugs/1486502
You received this bug notification because you are a member of Kernel Packages, which is subscribed to linux in Ubuntu.
  Thread Previous • Date Previous • • Thread Next