====================================== | [ 10.942636] ------------[ cut here ]------------ | [ 10.942977] DEBUG_LOCKS_WARN_ON(lock->magic != lock) | [ 10.942989] WARNING: CPU: 1 PID: 11 at kernel/locking/mutex.c:587 __mutex_lock (kernel/locking/mutex.c:587 kernel/locking/mutex.c:752) | [ 10.943996] Modules linked in: act_csum libcrc32c act_pedit cls_flower sch_prio [ 10.945132] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 10.946300] Workqueue: netns cleanup_net [ 10.946807] RIP: 0010:__mutex_lock (kernel/locking/mutex.c:587 kernel/locking/mutex.c:752) [ 10.947483] Code: ff 90 e8 a2 e7 70 ff 85 c0 74 23 8b 35 5c 8e db 00 85 f6 75 19 90 48 c7 c6 12 3a 2a 91 48 c7 c7 b5 ae 29 91 e8 00 04 26 ff 90 <0f> 0b 90 90 90 e9 03 fd ff ff 4c 8d 6d a0 4c 89 e7 4c 89 ee e8 d6 All code ======== 0: ff 90 e8 a2 e7 70 call *0x70e7a2e8(%rax) 6: ff 85 c0 74 23 8b incl -0x74dc8b40(%rbp) c: 35 5c 8e db 00 xor $0xdb8e5c,%eax 11: 85 f6 test %esi,%esi 13: 75 19 jne 0x2e 15: 90 nop 16: 48 c7 c6 12 3a 2a 91 mov $0xffffffff912a3a12,%rsi 1d: 48 c7 c7 b5 ae 29 91 mov $0xffffffff9129aeb5,%rdi 24: e8 00 04 26 ff call 0xffffffffff260429 29: 90 nop 2a:* 0f 0b ud2 <-- trapping instruction 2c: 90 nop 2d: 90 nop 2e: 90 nop 2f: e9 03 fd ff ff jmp 0xfffffffffffffd37 34: 4c 8d 6d a0 lea -0x60(%rbp),%r13 38: 4c 89 e7 mov %r12,%rdi 3b: 4c 89 ee mov %r13,%rsi 3e: e8 .byte 0xe8 3f: d6 (bad) Code starting with the faulting instruction =========================================== 0: 0f 0b ud2 2: 90 nop 3: 90 nop 4: 90 nop 5: e9 03 fd ff ff jmp 0xfffffffffffffd0d a: 4c 8d 6d a0 lea -0x60(%rbp),%r13 e: 4c 89 e7 mov %r12,%rdi 11: 4c 89 ee mov %r13,%rsi 14: e8 .byte 0xe8 15: d6 (bad) [ 10.949815] RSP: 0018:ffff9e3840063a10 EFLAGS: 00010286 [ 10.950523] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 00000000ffffdfff [ 10.951432] RDX: 0000000000000000 RSI: 00000000ffffffea RDI: 0000000000000001 [ 10.952389] RBP: ffff9e3840063a90 R08: ffffffff91778508 R09: 00000000ffffdfff [ 10.953238] R10: ffffffff91698520 R11: ffffffff9174ab30 R12: ffff906783ad2700 [ 10.954103] R13: 0000000000000000 R14: 0000000000000000 R15: ffff906782fbf800 [ 10.954963] FS: 0000000000000000(0000) GS:ffff9067bec80000(0000) knlGS:0000000000000000 [ 10.955881] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 10.956420] CR2: 00007fecc751c000 CR3: 0000000039660002 CR4: 0000000000770ef0 [ 10.956987] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 10.957374] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 10.957760] PKRU: 55555554 [ 10.957909] Call Trace: [ 10.958046] [ 10.958171] ? __warn (kernel/panic.c:677) [ 10.958367] ? __mutex_lock (kernel/locking/mutex.c:587 kernel/locking/mutex.c:752) [ 10.958592] ? report_bug (lib/bug.c:201 lib/bug.c:219) [ 10.958826] ? handle_bug (arch/x86/kernel/traps.c:238) [ 10.959031] ? exc_invalid_op (arch/x86/kernel/traps.c:259 (discriminator 1)) [ 10.959251] ? asm_exc_invalid_op (./arch/x86/include/asm/idtentry.h:568) [ 10.959496] ? __mutex_lock (kernel/locking/mutex.c:587 kernel/locking/mutex.c:752) [ 10.959719] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 10.960003] ? up_read (kernel/locking/rwsem.c:1622) [ 10.960197] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 10.960449] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 10.960736] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 10.961002] __tcf_action_put (net/sched/act_api.c:377) [ 10.961212] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 10.961428] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 10.961640] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 10.961930] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 10.962191] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 10.962443] tcf_proto_destroy (net/sched/cls_api.c:418) [ 10.962667] tcf_chain_flush (net/sched/cls_api.c:737) [ 10.962874] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 10.963081] tcf_block_put (net/sched/cls_api.c:1543) [ 10.963276] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 10.963540] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 10.963766] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 10.963975] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 10.964266] default_device_exit_batch (net/core/dev.c:11592) [ 10.964531] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 10.964743] ? process_one_work (kernel/workqueue.c:2606) [ 10.964983] process_one_work (kernel/workqueue.c:2633) [ 10.965213] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 10.965429] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 10.965674] kthread (kernel/kthread.c:388) [ 10.965858] ? __pfx_kthread (kernel/kthread.c:341) [ 10.966074] ret_from_fork (arch/x86/kernel/process.c:147) [ 10.966283] ? __pfx_kthread (kernel/kthread.c:341) [ 10.966494] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 10.969103] ---[ end trace 0000000000000000 ]--- | [ 36.970608] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 36.970949] rcu: 3-....: (26000 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=5880 | [ 36.971450] rcu: (t=26001 jiffies g=7477 q=1754 ncpus=4) [ 36.972186] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 36.972783] Workqueue: netns cleanup_net [ 36.972996] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 36.973292] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 36.974252] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 36.974536] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 36.974910] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 36.975284] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 36.975659] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 36.976034] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 36.976408] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 36.976833] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 36.977145] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 36.977519] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 36.977894] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 36.978268] PKRU: 55555554 [ 36.978416] Call Trace: [ 36.978554] [ 36.978669] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 36.978909] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 36.979154] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 36.979366] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 36.979574] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 36.979780] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 36.980021] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 36.980257] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 36.980543] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 36.980761] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 36.981017] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 36.981258] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 36.981488] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 36.981766] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 36.982031] [ 36.982149] [ 36.982267] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 36.982551] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 36.982813] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 36.983020] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 36.983219] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 36.983484] ? up_read (kernel/locking/rwsem.c:1622) [ 36.983662] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 36.983903] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 36.984168] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 36.984425] __tcf_action_put (net/sched/act_api.c:377) [ 36.984636] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 36.984851] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 36.985057] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 36.985339] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 36.985594] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 36.985841] tcf_proto_destroy (net/sched/cls_api.c:418) [ 36.986052] tcf_chain_flush (net/sched/cls_api.c:737) [ 36.986261] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 36.986468] tcf_block_put (net/sched/cls_api.c:1543) [ 36.986662] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 36.986902] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 36.987112] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 36.987316] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 36.987603] default_device_exit_batch (net/core/dev.c:11592) [ 36.987860] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 36.988054] ? process_one_work (kernel/workqueue.c:2606) [ 36.988280] process_one_work (kernel/workqueue.c:2633) [ 36.988497] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 36.988702] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 36.988931] kthread (kernel/kthread.c:388) [ 36.989105] ? __pfx_kthread (kernel/kthread.c:341) [ 36.989306] ret_from_fork (arch/x86/kernel/process.c:147) [ 36.989501] ? __pfx_kthread (kernel/kthread.c:341) [ 36.989702] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 46.144634] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 35145 jiffies s: 4157 root: 0x8/. | [ 46.145249] rcu: blocking rcu_node structures (internal RCU debug): | [ 46.145581] Sending NMI from CPU 2 to CPUs 3: | [ 46.145839] NMI backtrace for cpu 3 [ 46.145844] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 46.145845] Workqueue: netns cleanup_net [ 46.145848] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 46.145852] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 46.145853] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 46.145854] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 46.145855] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 46.145856] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 46.145857] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 46.145857] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 46.145860] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 46.145861] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 46.145862] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 46.145862] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 46.145863] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 46.145864] PKRU: 55555554 [ 46.145864] Call Trace: [ 46.145865] [ 46.145866] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 46.145870] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 46.145873] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 46.145875] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 46.145878] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 46.145880] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 46.145883] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 46.145886] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 46.145888] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 46.145890] [ 46.145890] [ 46.145891] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 46.145893] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 46.145895] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 46.145897] ? up_read (kernel/locking/rwsem.c:1622) [ 46.145899] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 46.145902] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 46.145904] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 46.145906] __tcf_action_put (net/sched/act_api.c:377) [ 46.145908] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 46.145910] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 46.145912] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 46.145917] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 46.145922] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 46.145927] tcf_proto_destroy (net/sched/cls_api.c:418) [ 46.145930] tcf_chain_flush (net/sched/cls_api.c:737) [ 46.145931] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 46.145933] tcf_block_put (net/sched/cls_api.c:1543) [ 46.145935] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 46.145939] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 46.145941] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 46.145943] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 46.145946] default_device_exit_batch (net/core/dev.c:11592) [ 46.145949] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 46.145951] ? process_one_work (kernel/workqueue.c:2606) [ 46.145954] process_one_work (kernel/workqueue.c:2633) [ 46.145956] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 46.145958] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 46.145960] kthread (kernel/kthread.c:388) [ 46.145962] ? __pfx_kthread (kernel/kthread.c:341) [ 46.145963] ret_from_fork (arch/x86/kernel/process.c:147) [ 46.145966] ? __pfx_kthread (kernel/kthread.c:341) [ 46.145967] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 46.145970] | [ 114.993609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 114.993965] rcu: 3-....: (103990 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=23619 | [ 114.994493] rcu: (t=104024 jiffies g=7477 q=1756 ncpus=4) [ 114.995239] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 114.995836] Workqueue: netns cleanup_net [ 114.996062] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 114.996367] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 114.997342] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 114.997644] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 114.998038] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 114.998417] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 114.998802] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 114.999191] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 114.999584] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 115.000030] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 115.000352] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 115.000752] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 115.001140] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 115.001539] PKRU: 55555554 [ 115.001695] Call Trace: [ 115.001851] [ 115.001966] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 115.002219] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 115.002473] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 115.002701] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 115.002919] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 115.003130] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 115.003388] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 115.003636] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 115.003943] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 115.004162] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 115.004425] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 115.004667] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 115.004898] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 115.005178] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 115.005442] [ 115.005562] [ 115.005681] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 115.005973] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 115.006240] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 115.006449] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 115.006648] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 115.006929] ? up_read (kernel/locking/rwsem.c:1622) [ 115.007107] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 115.007358] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 115.007640] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 115.007900] __tcf_action_put (net/sched/act_api.c:377) [ 115.008116] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 115.008341] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 115.008550] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 115.008836] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 115.009104] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 115.009365] tcf_proto_destroy (net/sched/cls_api.c:418) [ 115.009586] tcf_chain_flush (net/sched/cls_api.c:737) [ 115.009809] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 115.010027] tcf_block_put (net/sched/cls_api.c:1543) [ 115.010233] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 115.010484] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 115.010706] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 115.010908] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 115.011209] default_device_exit_batch (net/core/dev.c:11592) [ 115.011481] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 115.011690] ? process_one_work (kernel/workqueue.c:2606) [ 115.011924] process_one_work (kernel/workqueue.c:2633) [ 115.012159] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 115.012363] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 115.012591] kthread (kernel/kthread.c:388) [ 115.012767] ? __pfx_kthread (kernel/kthread.c:341) [ 115.012969] ret_from_fork (arch/x86/kernel/process.c:147) [ 115.013165] ? __pfx_kthread (kernel/kthread.c:341) [ 115.013366] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 144.448625] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 133449 jiffies s: 4157 root: 0x8/. | [ 144.449227] rcu: blocking rcu_node structures (internal RCU debug): | [ 144.449568] Sending NMI from CPU 2 to CPUs 3: | [ 144.449823] NMI backtrace for cpu 3 [ 144.449829] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 144.449831] Workqueue: netns cleanup_net [ 144.449836] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 144.449842] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 144.449843] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 144.449844] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 144.449846] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 144.449846] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 144.449847] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 144.449848] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 144.449851] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 144.449852] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 144.449853] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 144.449853] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 144.449854] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 144.449855] PKRU: 55555554 [ 144.449855] Call Trace: [ 144.449857] [ 144.449860] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 144.449864] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 144.449868] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 144.449870] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 144.449873] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 144.449875] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 144.449880] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 144.449882] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 144.449884] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 144.449886] [ 144.449887] [ 144.449887] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 144.449892] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 144.449894] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 144.449897] ? up_read (kernel/locking/rwsem.c:1622) [ 144.449900] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 144.449904] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 144.449906] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 144.449908] __tcf_action_put (net/sched/act_api.c:377) [ 144.449912] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 144.449915] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 144.449917] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 144.449923] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 144.449928] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 144.449933] tcf_proto_destroy (net/sched/cls_api.c:418) [ 144.449936] tcf_chain_flush (net/sched/cls_api.c:737) [ 144.449938] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 144.449940] tcf_block_put (net/sched/cls_api.c:1543) [ 144.449942] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 144.449946] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 144.449950] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 144.449953] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 144.449957] default_device_exit_batch (net/core/dev.c:11592) [ 144.449959] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 144.449962] ? process_one_work (kernel/workqueue.c:2606) [ 144.449966] process_one_work (kernel/workqueue.c:2633) [ 144.449968] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 144.449971] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 144.449972] kthread (kernel/kthread.c:388) [ 144.449975] ? __pfx_kthread (kernel/kthread.c:341) [ 144.449976] ret_from_fork (arch/x86/kernel/process.c:147) [ 144.449980] ? __pfx_kthread (kernel/kthread.c:341) [ 144.449981] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 144.449987] | [ 193.017609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 193.017952] rcu: 3-....: (181980 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=41355 | [ 193.018460] rcu: (t=182048 jiffies g=7477 q=1758 ncpus=4) [ 193.019203] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 193.019801] Workqueue: netns cleanup_net [ 193.020016] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 193.020311] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 193.021271] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 193.021548] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 193.021924] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 193.022299] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 193.022676] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 193.023050] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 193.023428] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 193.023855] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 193.024160] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 193.024534] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 193.024914] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 193.025290] PKRU: 55555554 [ 193.025437] Call Trace: [ 193.025574] [ 193.025693] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 193.025931] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 193.026175] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 193.026396] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 193.026605] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 193.026813] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 193.027053] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 193.027285] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 193.027571] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 193.027791] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 193.028047] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 193.028288] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 193.028517] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 193.028797] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 193.029063] [ 193.029180] [ 193.029298] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 193.029584] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 193.029847] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 193.030056] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 193.030255] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 193.030521] ? up_read (kernel/locking/rwsem.c:1622) [ 193.030704] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 193.030942] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 193.031207] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 193.031462] __tcf_action_put (net/sched/act_api.c:377) [ 193.031675] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 193.031891] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 193.032098] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 193.032379] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 193.032636] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 193.032883] tcf_proto_destroy (net/sched/cls_api.c:418) [ 193.033096] tcf_chain_flush (net/sched/cls_api.c:737) [ 193.033301] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 193.033511] tcf_block_put (net/sched/cls_api.c:1543) [ 193.033707] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 193.033947] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 193.034156] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 193.034351] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 193.034642] default_device_exit_batch (net/core/dev.c:11592) [ 193.034900] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 193.035095] ? process_one_work (kernel/workqueue.c:2606) [ 193.035319] process_one_work (kernel/workqueue.c:2633) [ 193.035536] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 193.035742] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 193.035971] kthread (kernel/kthread.c:388) [ 193.036146] ? __pfx_kthread (kernel/kthread.c:341) [ 193.036356] ret_from_fork (arch/x86/kernel/process.c:147) [ 193.036552] ? __pfx_kthread (kernel/kthread.c:341) [ 193.036755] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 242.752627] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 231753 jiffies s: 4157 root: 0x8/. | [ 242.753242] rcu: blocking rcu_node structures (internal RCU debug): | [ 242.753579] Sending NMI from CPU 2 to CPUs 3: | [ 242.753837] NMI backtrace for cpu 3 [ 242.753844] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 242.753846] Workqueue: netns cleanup_net [ 242.753852] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 242.753857] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 242.753859] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 242.753861] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 242.753862] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 242.753863] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 242.753864] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 242.753865] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 242.753867] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 242.753868] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 242.753869] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 242.753870] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 242.753871] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 242.753871] PKRU: 55555554 [ 242.753872] Call Trace: [ 242.753874] [ 242.753876] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 242.753880] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 242.753883] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 242.753886] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 242.753889] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 242.753891] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 242.753895] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 242.753898] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 242.753900] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 242.753902] [ 242.753902] [ 242.753903] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 242.753908] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 242.753909] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 242.753913] ? up_read (kernel/locking/rwsem.c:1622) [ 242.753915] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 242.753919] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 242.753921] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 242.753923] __tcf_action_put (net/sched/act_api.c:377) [ 242.753928] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 242.753930] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 242.753933] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 242.753938] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 242.753944] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 242.753949] tcf_proto_destroy (net/sched/cls_api.c:418) [ 242.753952] tcf_chain_flush (net/sched/cls_api.c:737) [ 242.753953] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 242.753955] tcf_block_put (net/sched/cls_api.c:1543) [ 242.753958] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 242.753962] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 242.753966] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 242.753968] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 242.753972] default_device_exit_batch (net/core/dev.c:11592) [ 242.753975] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 242.753977] ? process_one_work (kernel/workqueue.c:2606) [ 242.753981] process_one_work (kernel/workqueue.c:2633) [ 242.753983] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 242.753986] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 242.753987] kthread (kernel/kthread.c:388) [ 242.753989] ? __pfx_kthread (kernel/kthread.c:341) [ 242.753991] ret_from_fork (arch/x86/kernel/process.c:147) [ 242.753994] ? __pfx_kthread (kernel/kthread.c:341) [ 242.753995] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 242.754000] | [ 271.040619] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 271.040971] rcu: 3-....: (259970 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=59188 | [ 271.041479] rcu: (t=260071 jiffies g=7477 q=1760 ncpus=4) [ 271.042225] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 271.042822] Workqueue: netns cleanup_net [ 271.043039] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 271.043331] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 <8b> 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: 90 nop f: 90 nop 10: 90 nop 11: f3 0f 1e fa endbr64 15: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 1a: 41 55 push %r13 1c: 41 54 push %r12 1e: 55 push %rbp 1f: 48 89 fd mov %rdi,%rbp 22: 53 push %rbx 23: 66 90 xchg %ax,%ax 25: ba 01 00 00 00 mov $0x1,%edx 2a:* 8b 45 00 mov 0x0(%rbp),%eax <-- trapping instruction 2d: 85 c0 test %eax,%eax 2f: 75 14 jne 0x45 31: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 36: 85 c0 test %eax,%eax 38: 75 f0 jne 0x2a 3a: 5b pop %rbx 3b: 5d pop %rbp 3c: 41 5c pop %r12 3e: 41 5d pop %r13 Code starting with the faulting instruction =========================================== 0: 8b 45 00 mov 0x0(%rbp),%eax 3: 85 c0 test %eax,%eax 5: 75 14 jne 0x1b 7: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) c: 85 c0 test %eax,%eax e: 75 f0 jne 0x0 10: 5b pop %rbx 11: 5d pop %rbp 12: 41 5c pop %r12 14: 41 5d pop %r13 [ 271.044298] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 271.044574] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 271.044951] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 271.045325] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 271.045698] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 271.046078] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 271.046455] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 271.046881] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 271.047184] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 271.047558] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 271.047933] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 271.048306] PKRU: 55555554 [ 271.048453] Call Trace: [ 271.048593] [ 271.048714] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 271.048953] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 271.049198] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 271.049410] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 271.049618] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 271.049825] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 271.050064] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 271.050298] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 271.050582] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 271.050801] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 271.051057] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 271.051297] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 271.051526] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 271.051808] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 271.052072] [ 271.052190] [ 271.052307] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 271.052591] ? queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 271.052853] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 271.053060] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 271.053258] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 271.053524] ? up_read (kernel/locking/rwsem.c:1622) [ 271.053705] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 271.053945] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 271.054209] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 271.054464] __tcf_action_put (net/sched/act_api.c:377) [ 271.054676] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 271.054890] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 271.055097] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 271.055377] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 271.055633] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 271.055885] tcf_proto_destroy (net/sched/cls_api.c:418) [ 271.056098] tcf_chain_flush (net/sched/cls_api.c:737) [ 271.056305] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 271.056512] tcf_block_put (net/sched/cls_api.c:1543) [ 271.056707] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 271.056945] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 271.057156] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 271.057350] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 271.057639] default_device_exit_batch (net/core/dev.c:11592) [ 271.057895] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 271.058091] ? process_one_work (kernel/workqueue.c:2606) [ 271.058315] process_one_work (kernel/workqueue.c:2633) [ 271.058533] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 271.058737] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 271.058966] kthread (kernel/kthread.c:388) [ 271.059140] ? __pfx_kthread (kernel/kthread.c:341) [ 271.059342] ret_from_fork (arch/x86/kernel/process.c:147) [ 271.059537] ? __pfx_kthread (kernel/kthread.c:341) [ 271.059740] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 341.056626] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 330057 jiffies s: 4157 root: 0x8/. | [ 341.057266] rcu: blocking rcu_node structures (internal RCU debug): | [ 341.057616] Sending NMI from CPU 2 to CPUs 3: | [ 341.057868] NMI backtrace for cpu 3 [ 341.057874] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 341.057875] Workqueue: netns cleanup_net [ 341.057884] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 341.057890] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 341.057891] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 341.057892] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 341.057893] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 341.057895] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 341.057896] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 341.057896] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 341.057899] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 341.057900] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 341.057901] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 341.057902] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 341.057903] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 341.057904] PKRU: 55555554 [ 341.057904] Call Trace: [ 341.057906] [ 341.057908] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 341.057912] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 341.057916] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 341.057918] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 341.057922] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 341.057924] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 341.057928] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 341.057931] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 341.057933] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 341.057935] [ 341.057936] [ 341.057937] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 341.057941] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 341.057943] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 341.057946] ? up_read (kernel/locking/rwsem.c:1622) [ 341.057949] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 341.057953] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 341.057954] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 341.057956] __tcf_action_put (net/sched/act_api.c:377) [ 341.057961] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 341.057963] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 341.057965] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 341.057971] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 341.057976] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 341.057981] tcf_proto_destroy (net/sched/cls_api.c:418) [ 341.057984] tcf_chain_flush (net/sched/cls_api.c:737) [ 341.057985] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 341.057987] tcf_block_put (net/sched/cls_api.c:1543) [ 341.057990] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 341.057994] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 341.057998] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 341.058000] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 341.058004] default_device_exit_batch (net/core/dev.c:11592) [ 341.058007] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 341.058009] ? process_one_work (kernel/workqueue.c:2606) [ 341.058013] process_one_work (kernel/workqueue.c:2633) [ 341.058016] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 341.058018] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 341.058020] kthread (kernel/kthread.c:388) [ 341.058022] ? __pfx_kthread (kernel/kthread.c:341) [ 341.058024] ret_from_fork (arch/x86/kernel/process.c:147) [ 341.058027] ? __pfx_kthread (kernel/kthread.c:341) [ 341.058028] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 341.058033] | [ 349.062609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 349.062940] rcu: 3-....: (337959 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=76814 | [ 349.063446] rcu: (t=338093 jiffies g=7477 q=1769 ncpus=4) [ 349.064180] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 349.064774] Workqueue: netns cleanup_net [ 349.064988] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 349.065282] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 349.066246] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 349.066524] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 349.066897] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 349.067271] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 349.067646] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 349.068019] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 349.068392] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 349.068817] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 349.069120] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 349.069494] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 349.069872] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 349.070245] PKRU: 55555554 [ 349.070391] Call Trace: [ 349.070528] [ 349.070644] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 349.070882] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 349.071125] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 349.071338] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 349.071545] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 349.071752] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 349.071992] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 349.072224] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 349.072508] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 349.072728] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 349.072983] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 349.073223] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 349.073451] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 349.073729] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 349.073993] [ 349.074110] [ 349.074227] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 349.074510] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 349.074771] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 349.074977] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 349.075176] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 349.075441] ? up_read (kernel/locking/rwsem.c:1622) [ 349.075627] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 349.075867] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 349.076129] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 349.076384] __tcf_action_put (net/sched/act_api.c:377) [ 349.076592] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 349.076806] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 349.077012] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 349.077291] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 349.077548] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 349.077794] tcf_proto_destroy (net/sched/cls_api.c:418) [ 349.078005] tcf_chain_flush (net/sched/cls_api.c:737) [ 349.078211] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 349.078418] tcf_block_put (net/sched/cls_api.c:1543) [ 349.078611] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 349.078850] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 349.079059] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 349.079252] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 349.079540] default_device_exit_batch (net/core/dev.c:11592) [ 349.079796] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 349.079991] ? process_one_work (kernel/workqueue.c:2606) [ 349.080214] process_one_work (kernel/workqueue.c:2633) [ 349.080432] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 349.080635] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 349.080862] kthread (kernel/kthread.c:388) [ 349.081036] ? __pfx_kthread (kernel/kthread.c:341) [ 349.081238] ret_from_fork (arch/x86/kernel/process.c:147) [ 349.081432] ? __pfx_kthread (kernel/kthread.c:341) [ 349.081634] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 349.081847] | [ 427.085609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 427.085949] rcu: 3-....: (415964 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=94790 | [ 427.086457] rcu: (t=416116 jiffies g=7477 q=1769 ncpus=4) [ 427.087195] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 427.087790] Workqueue: netns cleanup_net [ 427.088004] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 427.088298] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 <85> c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: f3 0f 1e fa endbr64 12: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 17: 41 55 push %r13 19: 41 54 push %r12 1b: 55 push %rbp 1c: 48 89 fd mov %rdi,%rbp 1f: 53 push %rbx 20: 66 90 xchg %ax,%ax 22: ba 01 00 00 00 mov $0x1,%edx 27: 8b 45 00 mov 0x0(%rbp),%eax 2a:* 85 c0 test %eax,%eax <-- trapping instruction 2c: 75 14 jne 0x42 2e: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 33: 85 c0 test %eax,%eax 35: 75 f0 jne 0x27 37: 5b pop %rbx 38: 5d pop %rbp 39: 41 5c pop %r12 3b: 41 5d pop %r13 3d: c3 ret 3e: cc int3 3f: cc int3 Code starting with the faulting instruction =========================================== 0: 85 c0 test %eax,%eax 2: 75 14 jne 0x18 4: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 9: 85 c0 test %eax,%eax b: 75 f0 jne 0xfffffffffffffffd d: 5b pop %rbx e: 5d pop %rbp f: 41 5c pop %r12 11: 41 5d pop %r13 13: c3 ret 14: cc int3 15: cc int3 [ 427.089258] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 427.089536] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 427.089908] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 427.090282] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 427.090658] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 427.091030] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 427.091403] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 427.091829] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 427.092133] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 427.092505] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 427.092878] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 427.093252] PKRU: 55555554 [ 427.093398] Call Trace: [ 427.093534] [ 427.093651] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 427.093889] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 427.094132] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 427.094344] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 427.094551] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 427.094758] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 427.094998] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 427.095238] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 427.095520] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 427.095739] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 427.095995] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 427.096235] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 427.096464] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 427.096745] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 427.097010] [ 427.097128] [ 427.097246] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 427.097530] ? queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 427.097792] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 427.098000] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 427.098199] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 427.098468] ? up_read (kernel/locking/rwsem.c:1622) [ 427.098647] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 427.098891] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 427.099155] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 427.099412] __tcf_action_put (net/sched/act_api.c:377) [ 427.099622] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 427.099837] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 427.100043] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 427.100325] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 427.100580] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 427.100828] tcf_proto_destroy (net/sched/cls_api.c:418) [ 427.101038] tcf_chain_flush (net/sched/cls_api.c:737) [ 427.101246] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 427.101455] tcf_block_put (net/sched/cls_api.c:1543) [ 427.101652] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 427.101891] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 427.102107] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 427.102302] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 427.102588] default_device_exit_batch (net/core/dev.c:11592) [ 427.102846] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 427.103042] ? process_one_work (kernel/workqueue.c:2606) [ 427.103266] process_one_work (kernel/workqueue.c:2633) [ 427.103485] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 427.103689] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 427.103918] kthread (kernel/kthread.c:388) [ 427.104092] ? __pfx_kthread (kernel/kthread.c:341) [ 427.104294] ret_from_fork (arch/x86/kernel/process.c:147) [ 427.104490] ? __pfx_kthread (kernel/kthread.c:341) [ 427.104693] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 439.360629] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 428361 jiffies s: 4157 root: 0x8/. | [ 439.361237] rcu: blocking rcu_node structures (internal RCU debug): | [ 439.361567] Sending NMI from CPU 2 to CPUs 3: | [ 439.361827] NMI backtrace for cpu 3 [ 439.361832] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 439.361833] Workqueue: netns cleanup_net [ 439.361838] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 439.361842] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 439.361844] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 439.361845] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 439.361846] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 439.361847] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 439.361848] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 439.361849] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 439.361851] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 439.361852] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 439.361853] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 439.361854] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 439.361854] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 439.361855] PKRU: 55555554 [ 439.361856] Call Trace: [ 439.361858] [ 439.361859] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 439.361863] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 439.361867] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 439.361869] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 439.361872] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 439.361874] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 439.361878] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 439.361880] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 439.361882] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 439.361885] [ 439.361885] [ 439.361886] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 439.361889] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 439.361891] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 439.361894] ? up_read (kernel/locking/rwsem.c:1622) [ 439.361896] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 439.361900] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 439.361901] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 439.361903] __tcf_action_put (net/sched/act_api.c:377) [ 439.361907] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 439.361909] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 439.361911] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 439.361916] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 439.361922] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 439.361927] tcf_proto_destroy (net/sched/cls_api.c:418) [ 439.361929] tcf_chain_flush (net/sched/cls_api.c:737) [ 439.361930] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 439.361932] tcf_block_put (net/sched/cls_api.c:1543) [ 439.361934] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 439.361938] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 439.361942] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 439.361944] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 439.361947] default_device_exit_batch (net/core/dev.c:11592) [ 439.361950] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 439.361952] ? process_one_work (kernel/workqueue.c:2606) [ 439.361955] process_one_work (kernel/workqueue.c:2633) [ 439.361958] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 439.361960] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 439.361962] kthread (kernel/kthread.c:388) [ 439.361964] ? __pfx_kthread (kernel/kthread.c:341) [ 439.361965] ret_from_fork (arch/x86/kernel/process.c:147) [ 439.361968] ? __pfx_kthread (kernel/kthread.c:341) [ 439.361969] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 439.361974] | [ 505.108609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 505.108950] rcu: 3-....: (493954 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=112976 | [ 505.109466] rcu: (t=494139 jiffies g=7477 q=1770 ncpus=4) [ 505.110207] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 505.110804] Workqueue: netns cleanup_net [ 505.111019] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 505.111315] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 <8b> 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: 90 nop f: 90 nop 10: 90 nop 11: f3 0f 1e fa endbr64 15: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 1a: 41 55 push %r13 1c: 41 54 push %r12 1e: 55 push %rbp 1f: 48 89 fd mov %rdi,%rbp 22: 53 push %rbx 23: 66 90 xchg %ax,%ax 25: ba 01 00 00 00 mov $0x1,%edx 2a:* 8b 45 00 mov 0x0(%rbp),%eax <-- trapping instruction 2d: 85 c0 test %eax,%eax 2f: 75 14 jne 0x45 31: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 36: 85 c0 test %eax,%eax 38: 75 f0 jne 0x2a 3a: 5b pop %rbx 3b: 5d pop %rbp 3c: 41 5c pop %r12 3e: 41 5d pop %r13 Code starting with the faulting instruction =========================================== 0: 8b 45 00 mov 0x0(%rbp),%eax 3: 85 c0 test %eax,%eax 5: 75 14 jne 0x1b 7: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) c: 85 c0 test %eax,%eax e: 75 f0 jne 0x0 10: 5b pop %rbx 11: 5d pop %rbp 12: 41 5c pop %r12 14: 41 5d pop %r13 [ 505.112281] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 505.112560] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 505.112934] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 505.113309] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 505.113684] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 505.114059] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 505.114433] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 505.114865] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 505.115172] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 505.115545] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 505.115922] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 505.116296] PKRU: 55555554 [ 505.116444] Call Trace: [ 505.116580] [ 505.116697] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 505.116936] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 505.117180] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 505.117393] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 505.117601] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 505.117810] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 505.118050] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 505.118284] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 505.118567] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 505.118787] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 505.119043] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 505.119284] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 505.119513] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 505.119796] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 505.120061] [ 505.120178] [ 505.120296] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 505.120580] ? queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 505.120842] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 505.121050] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 505.121249] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 505.121515] ? up_read (kernel/locking/rwsem.c:1622) [ 505.121696] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 505.121937] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 505.122202] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 505.122458] __tcf_action_put (net/sched/act_api.c:377) [ 505.122670] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 505.122886] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 505.123094] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 505.123375] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 505.123632] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 505.123881] tcf_proto_destroy (net/sched/cls_api.c:418) [ 505.124093] tcf_chain_flush (net/sched/cls_api.c:737) [ 505.124301] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 505.124510] tcf_block_put (net/sched/cls_api.c:1543) [ 505.124710] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 505.124951] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 505.125161] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 505.125357] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 505.125646] default_device_exit_batch (net/core/dev.c:11592) [ 505.125905] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 505.126101] ? process_one_work (kernel/workqueue.c:2606) [ 505.126326] process_one_work (kernel/workqueue.c:2633) [ 505.126545] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 505.126751] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 505.126981] kthread (kernel/kthread.c:388) [ 505.127155] ? __pfx_kthread (kernel/kthread.c:341) [ 505.127357] ret_from_fork (arch/x86/kernel/process.c:147) [ 505.127552] ? __pfx_kthread (kernel/kthread.c:341) [ 505.127755] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 537.664621] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 526665 jiffies s: 4157 root: 0x8/. | [ 537.665231] rcu: blocking rcu_node structures (internal RCU debug): | [ 537.665564] Sending NMI from CPU 2 to CPUs 3: | [ 537.665818] NMI backtrace for cpu 3 [ 537.665826] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 537.665828] Workqueue: netns cleanup_net [ 537.665834] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 537.665840] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 537.665841] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 537.665842] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 537.665843] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 537.665844] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 537.665846] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 537.665847] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 537.665849] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 537.665850] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 537.665851] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 537.665852] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 537.665853] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 537.665854] PKRU: 55555554 [ 537.665854] Call Trace: [ 537.665856] [ 537.665858] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 537.665862] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 537.665866] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 537.665868] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 537.665872] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 537.665874] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 537.665878] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 537.665881] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 537.665883] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 537.665885] [ 537.665886] [ 537.665886] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 537.665891] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 537.665892] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 537.665896] ? up_read (kernel/locking/rwsem.c:1622) [ 537.665898] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 537.665902] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 537.665904] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 537.665906] __tcf_action_put (net/sched/act_api.c:377) [ 537.665909] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 537.665912] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 537.665914] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 537.665920] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 537.665925] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 537.665930] tcf_proto_destroy (net/sched/cls_api.c:418) [ 537.665933] tcf_chain_flush (net/sched/cls_api.c:737) [ 537.665934] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 537.665936] tcf_block_put (net/sched/cls_api.c:1543) [ 537.665939] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 537.665943] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 537.665947] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 537.665949] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 537.665953] default_device_exit_batch (net/core/dev.c:11592) [ 537.665955] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 537.665958] ? process_one_work (kernel/workqueue.c:2606) [ 537.665960] process_one_work (kernel/workqueue.c:2633) [ 537.665963] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 537.665965] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 537.665967] kthread (kernel/kthread.c:388) [ 537.665969] ? __pfx_kthread (kernel/kthread.c:341) [ 537.665971] ret_from_fork (arch/x86/kernel/process.c:147) [ 537.665974] ? __pfx_kthread (kernel/kthread.c:341) [ 537.665975] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 537.665980] | [ 583.131609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 583.131961] rcu: 3-....: (571944 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=131009 | [ 583.132488] rcu: (t=572162 jiffies g=7477 q=1771 ncpus=4) [ 583.133258] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 583.133870] Workqueue: netns cleanup_net [ 583.134099] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 583.134405] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 583.135396] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 583.135681] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 583.136066] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 583.136450] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 583.136835] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 583.137219] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 583.137604] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 583.138039] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 583.138351] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 583.138737] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 583.139120] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 583.139503] PKRU: 55555554 [ 583.139656] Call Trace: [ 583.139799] [ 583.139926] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 583.140172] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 583.140421] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 583.140643] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 583.140858] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 583.141072] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 583.141320] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 583.141563] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 583.141859] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 583.142085] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 583.142351] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 583.142601] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 583.142839] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 583.143129] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 583.143403] [ 583.143525] [ 583.143648] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 583.143944] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 583.144221] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 583.144440] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 583.144647] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 583.144924] ? up_read (kernel/locking/rwsem.c:1622) [ 583.145109] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 583.145357] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 583.145632] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 583.145897] __tcf_action_put (net/sched/act_api.c:377) [ 583.146117] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 583.146341] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 583.146555] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 583.146849] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 583.147113] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 583.147370] tcf_proto_destroy (net/sched/cls_api.c:418) [ 583.147589] tcf_chain_flush (net/sched/cls_api.c:737) [ 583.147805] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 583.148019] tcf_block_put (net/sched/cls_api.c:1543) [ 583.148220] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 583.148469] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 583.148687] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 583.148889] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 583.149185] default_device_exit_batch (net/core/dev.c:11592) [ 583.149452] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 583.149656] ? process_one_work (kernel/workqueue.c:2606) [ 583.149887] process_one_work (kernel/workqueue.c:2633) [ 583.150115] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 583.150325] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 583.150562] kthread (kernel/kthread.c:388) [ 583.150743] ? __pfx_kthread (kernel/kthread.c:341) [ 583.150952] ret_from_fork (arch/x86/kernel/process.c:147) [ 583.151154] ? __pfx_kthread (kernel/kthread.c:341) [ 583.151362] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 635.968620] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 624969 jiffies s: 4157 root: 0x8/. | [ 635.969230] rcu: blocking rcu_node structures (internal RCU debug): | [ 635.969560] Sending NMI from CPU 2 to CPUs 3: | [ 635.969811] NMI backtrace for cpu 3 [ 635.969816] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 635.969818] Workqueue: netns cleanup_net [ 635.969824] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 635.969829] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 635.969831] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 635.969832] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 635.969833] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 635.969834] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 635.969835] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 635.969836] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 635.969839] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 635.969840] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 635.969840] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 635.969841] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 635.969842] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 635.969843] PKRU: 55555554 [ 635.969843] Call Trace: [ 635.969846] [ 635.969848] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 635.969852] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 635.969855] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 635.969858] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 635.969861] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 635.969863] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 635.969867] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 635.969870] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 635.969872] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 635.969874] [ 635.969875] [ 635.969875] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 635.969880] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 635.969881] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 635.969885] ? up_read (kernel/locking/rwsem.c:1622) [ 635.969888] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 635.969891] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 635.969893] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 635.969895] __tcf_action_put (net/sched/act_api.c:377) [ 635.969899] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 635.969901] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 635.969903] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 635.969909] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 635.969914] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 635.969919] tcf_proto_destroy (net/sched/cls_api.c:418) [ 635.969922] tcf_chain_flush (net/sched/cls_api.c:737) [ 635.969923] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 635.969925] tcf_block_put (net/sched/cls_api.c:1543) [ 635.969928] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 635.969932] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 635.969936] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 635.969938] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 635.969942] default_device_exit_batch (net/core/dev.c:11592) [ 635.969945] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 635.969947] ? process_one_work (kernel/workqueue.c:2606) [ 635.969950] process_one_work (kernel/workqueue.c:2633) [ 635.969953] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 635.969955] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 635.969957] kthread (kernel/kthread.c:388) [ 635.969959] ? __pfx_kthread (kernel/kthread.c:341) [ 635.969961] ret_from_fork (arch/x86/kernel/process.c:147) [ 635.969963] ? __pfx_kthread (kernel/kthread.c:341) [ 635.969965] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 635.969969] | [ 661.155609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 661.155961] rcu: 3-....: (649934 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=148990 | [ 661.156484] rcu: (t=650186 jiffies g=7477 q=1771 ncpus=4) [ 661.157251] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 661.157870] Workqueue: netns cleanup_net [ 661.158092] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 661.158391] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 <8b> 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: 90 nop f: 90 nop 10: 90 nop 11: f3 0f 1e fa endbr64 15: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 1a: 41 55 push %r13 1c: 41 54 push %r12 1e: 55 push %rbp 1f: 48 89 fd mov %rdi,%rbp 22: 53 push %rbx 23: 66 90 xchg %ax,%ax 25: ba 01 00 00 00 mov $0x1,%edx 2a:* 8b 45 00 mov 0x0(%rbp),%eax <-- trapping instruction 2d: 85 c0 test %eax,%eax 2f: 75 14 jne 0x45 31: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 36: 85 c0 test %eax,%eax 38: 75 f0 jne 0x2a 3a: 5b pop %rbx 3b: 5d pop %rbp 3c: 41 5c pop %r12 3e: 41 5d pop %r13 Code starting with the faulting instruction =========================================== 0: 8b 45 00 mov 0x0(%rbp),%eax 3: 85 c0 test %eax,%eax 5: 75 14 jne 0x1b 7: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) c: 85 c0 test %eax,%eax e: 75 f0 jne 0x0 10: 5b pop %rbx 11: 5d pop %rbp 12: 41 5c pop %r12 14: 41 5d pop %r13 [ 661.159377] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 661.159661] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 661.160045] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 661.160428] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 661.160812] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 661.161193] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 661.161579] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 661.162014] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 661.162323] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 661.162710] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 661.163093] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 661.163477] PKRU: 55555554 [ 661.163629] Call Trace: [ 661.163769] [ 661.163895] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 661.164140] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 661.164389] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 661.164609] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 661.164825] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 661.165038] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 661.165286] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 661.165527] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 661.165820] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 661.166047] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 661.166312] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 661.166559] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 661.166797] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 661.167083] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 661.167354] [ 661.167477] [ 661.167599] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 661.167893] ? queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 661.168163] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 661.168379] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 661.168584] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 661.168877] ? up_read (kernel/locking/rwsem.c:1622) [ 661.169061] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 661.169309] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 661.169580] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 661.169845] __tcf_action_put (net/sched/act_api.c:377) [ 661.170059] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 661.170284] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 661.170497] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 661.170787] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 661.171051] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 661.171306] tcf_proto_destroy (net/sched/cls_api.c:418) [ 661.171524] tcf_chain_flush (net/sched/cls_api.c:737) [ 661.171738] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 661.171950] tcf_block_put (net/sched/cls_api.c:1543) [ 661.172150] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 661.172400] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 661.172614] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 661.172816] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 661.173115] default_device_exit_batch (net/core/dev.c:11592) [ 661.173380] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 661.173580] ? process_one_work (kernel/workqueue.c:2606) [ 661.173818] process_one_work (kernel/workqueue.c:2633) [ 661.174042] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 661.174253] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 661.174491] kthread (kernel/kthread.c:388) [ 661.174672] ? __pfx_kthread (kernel/kthread.c:341) [ 661.174880] ret_from_fork (arch/x86/kernel/process.c:147) [ 661.175082] ? __pfx_kthread (kernel/kthread.c:341) [ 661.175291] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 734.272633] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 723273 jiffies s: 4157 root: 0x8/. | [ 734.273479] rcu: blocking rcu_node structures (internal RCU debug): | [ 734.273949] Sending NMI from CPU 2 to CPUs 3: | [ 734.274295] NMI backtrace for cpu 3 [ 734.274303] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 734.274305] Workqueue: netns cleanup_net [ 734.274311] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 734.274316] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 734.274317] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 734.274319] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 734.274320] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 734.274321] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 734.274322] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 734.274323] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 734.274325] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 734.274326] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 734.274327] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 734.274328] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 734.274328] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 734.274329] PKRU: 55555554 [ 734.274330] Call Trace: [ 734.274332] [ 734.274335] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 734.274339] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 734.274343] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 734.274345] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 734.274348] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 734.274350] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 734.274354] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 734.274357] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 734.274359] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 734.274361] [ 734.274362] [ 734.274362] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 734.274367] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 734.274368] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 734.274373] ? up_read (kernel/locking/rwsem.c:1622) [ 734.274376] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 734.274379] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 734.274381] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 734.274383] __tcf_action_put (net/sched/act_api.c:377) [ 734.274387] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 734.274389] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 734.274391] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 734.274397] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 734.274403] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 734.274408] tcf_proto_destroy (net/sched/cls_api.c:418) [ 734.274411] tcf_chain_flush (net/sched/cls_api.c:737) [ 734.274412] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 734.274414] tcf_block_put (net/sched/cls_api.c:1543) [ 734.274416] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 734.274420] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 734.274424] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 734.274427] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 734.274430] default_device_exit_batch (net/core/dev.c:11592) [ 734.274433] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 734.274436] ? process_one_work (kernel/workqueue.c:2606) [ 734.274438] process_one_work (kernel/workqueue.c:2633) [ 734.274441] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 734.274443] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 734.274445] kthread (kernel/kthread.c:388) [ 734.274447] ? __pfx_kthread (kernel/kthread.c:341) [ 734.274449] ret_from_fork (arch/x86/kernel/process.c:147) [ 734.274451] ? __pfx_kthread (kernel/kthread.c:341) [ 734.274453] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 734.274458] | [ 739.179608] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 739.179955] rcu: 3-....: (727924 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=166783 | [ 739.180483] rcu: (t=728210 jiffies g=7477 q=1771 ncpus=4) [ 739.181244] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 739.181853] Workqueue: netns cleanup_net [ 739.182072] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 739.182373] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 739.183361] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 739.183655] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 739.184039] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 739.184423] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 739.184808] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 739.185191] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 739.185573] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 739.186009] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 739.186322] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 739.186706] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 739.187090] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 739.187473] PKRU: 55555554 [ 739.187625] Call Trace: [ 739.187764] [ 739.187883] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 739.188129] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 739.188379] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 739.188598] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 739.188812] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 739.189025] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 739.189273] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 739.189516] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 739.189811] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 739.190035] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 739.190299] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 739.190548] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 739.190786] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 739.191074] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 739.191347] [ 739.191468] [ 739.191588] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 739.191885] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 739.192154] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 739.192368] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 739.192574] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 739.192849] ? up_read (kernel/locking/rwsem.c:1622) [ 739.193034] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 739.193281] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 739.193562] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 739.193826] __tcf_action_put (net/sched/act_api.c:377) [ 739.194041] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 739.194263] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 739.194478] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 739.194769] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 739.195031] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 739.195286] tcf_proto_destroy (net/sched/cls_api.c:418) [ 739.195504] tcf_chain_flush (net/sched/cls_api.c:737) [ 739.195719] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 739.195933] tcf_block_put (net/sched/cls_api.c:1543) [ 739.196134] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 739.196386] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 739.196604] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 739.196806] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 739.197102] default_device_exit_batch (net/core/dev.c:11592) [ 739.197368] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 739.197569] ? process_one_work (kernel/workqueue.c:2606) [ 739.197801] process_one_work (kernel/workqueue.c:2633) [ 739.198026] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 739.198237] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 739.198473] kthread (kernel/kthread.c:388) [ 739.198653] ? __pfx_kthread (kernel/kthread.c:341) [ 739.198862] ret_from_fork (arch/x86/kernel/process.c:147) [ 739.199063] ? __pfx_kthread (kernel/kthread.c:341) [ 739.199271] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 739.199500] | [ 817.203609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 817.203951] rcu: 3-....: (805929 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=184720 | [ 817.204465] rcu: (t=806234 jiffies g=7477 q=1771 ncpus=4) [ 817.205205] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 817.205799] Workqueue: netns cleanup_net [ 817.206014] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 817.206309] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 817.207270] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 817.207550] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 817.207923] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 817.208297] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 817.208673] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 817.209048] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 817.209423] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 817.209849] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 817.210154] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 817.210528] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 817.210904] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 817.211277] PKRU: 55555554 [ 817.211424] Call Trace: [ 817.211560] [ 817.211677] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 817.211916] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 817.212160] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 817.212371] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 817.212580] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 817.212788] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 817.213033] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 817.213268] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 817.213553] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 817.213771] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 817.214027] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 817.214269] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 817.214498] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 817.214779] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 817.215045] [ 817.215163] [ 817.215281] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 817.215573] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 817.215837] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 817.216044] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 817.216243] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 817.216509] ? up_read (kernel/locking/rwsem.c:1622) [ 817.216690] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 817.216931] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 817.217196] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 817.217452] __tcf_action_put (net/sched/act_api.c:377) [ 817.217662] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 817.217877] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 817.218084] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 817.218365] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 817.218620] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 817.218869] tcf_proto_destroy (net/sched/cls_api.c:418) [ 817.219080] tcf_chain_flush (net/sched/cls_api.c:737) [ 817.219288] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 817.219495] tcf_block_put (net/sched/cls_api.c:1543) [ 817.219693] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 817.219933] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 817.220143] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 817.220338] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 817.220627] default_device_exit_batch (net/core/dev.c:11592) [ 817.220885] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 817.221080] ? process_one_work (kernel/workqueue.c:2606) [ 817.221304] process_one_work (kernel/workqueue.c:2633) [ 817.221522] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 817.221727] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 817.221955] kthread (kernel/kthread.c:388) [ 817.222129] ? __pfx_kthread (kernel/kthread.c:341) [ 817.222331] ret_from_fork (arch/x86/kernel/process.c:147) [ 817.222527] ? __pfx_kthread (kernel/kthread.c:341) [ 817.222730] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 832.576635] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 821577 jiffies s: 4157 root: 0x8/. | [ 832.577544] rcu: blocking rcu_node structures (internal RCU debug): | [ 832.578047] Sending NMI from CPU 2 to CPUs 3: | [ 832.578399] NMI backtrace for cpu 3 [ 832.578404] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 832.578405] Workqueue: netns cleanup_net [ 832.578409] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 832.578412] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 832.578413] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 832.578414] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 832.578415] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 832.578416] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 832.578417] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 832.578418] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 832.578420] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 832.578421] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 832.578422] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 832.578423] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 832.578424] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 832.578424] PKRU: 55555554 [ 832.578425] Call Trace: [ 832.578426] [ 832.578428] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 832.578433] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 832.578436] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 832.578438] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 832.578441] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 832.578443] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 832.578446] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 832.578449] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 832.578451] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 832.578453] [ 832.578454] [ 832.578454] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 832.578457] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 832.578458] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 832.578461] ? up_read (kernel/locking/rwsem.c:1622) [ 832.578464] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 832.578467] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 832.578468] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 832.578471] __tcf_action_put (net/sched/act_api.c:377) [ 832.578473] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 832.578475] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 832.578477] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 832.578482] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 832.578488] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 832.578492] tcf_proto_destroy (net/sched/cls_api.c:418) [ 832.578495] tcf_chain_flush (net/sched/cls_api.c:737) [ 832.578496] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 832.578498] tcf_block_put (net/sched/cls_api.c:1543) [ 832.578500] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 832.578504] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 832.578506] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 832.578509] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 832.578511] default_device_exit_batch (net/core/dev.c:11592) [ 832.578514] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 832.578516] ? process_one_work (kernel/workqueue.c:2606) [ 832.578518] process_one_work (kernel/workqueue.c:2633) [ 832.578520] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 832.578523] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 832.578524] kthread (kernel/kthread.c:388) [ 832.578526] ? __pfx_kthread (kernel/kthread.c:341) [ 832.578527] ret_from_fork (arch/x86/kernel/process.c:147) [ 832.578530] ? __pfx_kthread (kernel/kthread.c:341) [ 832.578531] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 832.578535] | [ 895.226608] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 895.226943] rcu: 3-....: (883919 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=202662 | [ 895.227456] rcu: (t=884257 jiffies g=7477 q=1771 ncpus=4) [ 895.228194] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 895.228787] Workqueue: netns cleanup_net [ 895.229003] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 895.229296] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 895.230263] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 895.230540] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 895.230913] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 895.231285] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 895.231660] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 895.232035] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 895.232410] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 895.232840] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 895.233144] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 895.233518] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 895.233891] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 895.234264] PKRU: 55555554 [ 895.234411] Call Trace: [ 895.234549] [ 895.234664] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 895.234903] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 895.235148] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 895.235359] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 895.235567] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 895.235774] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 895.236015] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 895.236253] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 895.236536] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 895.236755] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 895.237011] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 895.237253] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 895.237484] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 895.237763] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 895.238027] [ 895.238146] [ 895.238264] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 895.238547] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 895.238809] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 895.239017] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 895.239216] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 895.239482] ? up_read (kernel/locking/rwsem.c:1622) [ 895.239660] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 895.239902] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 895.240167] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 895.240423] __tcf_action_put (net/sched/act_api.c:377) [ 895.240633] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 895.240848] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 895.241054] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 895.241336] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 895.241591] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 895.241837] tcf_proto_destroy (net/sched/cls_api.c:418) [ 895.242050] tcf_chain_flush (net/sched/cls_api.c:737) [ 895.242258] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 895.242464] tcf_block_put (net/sched/cls_api.c:1543) [ 895.242666] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 895.242920] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 895.243131] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 895.243326] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 895.243613] default_device_exit_batch (net/core/dev.c:11592) [ 895.243871] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 895.244066] ? process_one_work (kernel/workqueue.c:2606) [ 895.244291] process_one_work (kernel/workqueue.c:2633) [ 895.244509] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 895.244712] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 895.244943] kthread (kernel/kthread.c:388) [ 895.245116] ? __pfx_kthread (kernel/kthread.c:341) [ 895.245318] ret_from_fork (arch/x86/kernel/process.c:147) [ 895.245514] ? __pfx_kthread (kernel/kthread.c:341) [ 895.245716] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 930.880621] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 919881 jiffies s: 4157 root: 0x8/. | [ 930.881244] rcu: blocking rcu_node structures (internal RCU debug): | [ 930.881574] Sending NMI from CPU 2 to CPUs 3: | [ 930.881826] NMI backtrace for cpu 3 [ 930.881831] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 930.881832] Workqueue: netns cleanup_net [ 930.881836] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 930.881841] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 930.881842] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 930.881843] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 930.881844] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 930.881845] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 930.881845] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 930.881846] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 930.881848] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 930.881849] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 930.881850] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 930.881851] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 930.881852] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 930.881852] PKRU: 55555554 [ 930.881853] Call Trace: [ 930.881855] [ 930.881856] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 930.881860] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 930.881863] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 930.881865] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 930.881868] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 930.881870] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 930.881874] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 930.881876] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 930.881878] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 930.881881] [ 930.881881] [ 930.881882] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 930.881885] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 930.881886] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 930.881889] ? up_read (kernel/locking/rwsem.c:1622) [ 930.881891] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 930.881895] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 930.881897] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 930.881899] __tcf_action_put (net/sched/act_api.c:377) [ 930.881902] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 930.881905] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 930.881907] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 930.881912] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 930.881917] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 930.881922] tcf_proto_destroy (net/sched/cls_api.c:418) [ 930.881925] tcf_chain_flush (net/sched/cls_api.c:737) [ 930.881926] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 930.881928] tcf_block_put (net/sched/cls_api.c:1543) [ 930.881930] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 930.881934] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 930.881937] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 930.881940] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 930.881943] default_device_exit_batch (net/core/dev.c:11592) [ 930.881945] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 930.881947] ? process_one_work (kernel/workqueue.c:2606) [ 930.881950] process_one_work (kernel/workqueue.c:2633) [ 930.881952] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 930.881954] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 930.881956] kthread (kernel/kthread.c:388) [ 930.881958] ? __pfx_kthread (kernel/kthread.c:341) [ 930.881959] ret_from_fork (arch/x86/kernel/process.c:147) [ 930.881962] ? __pfx_kthread (kernel/kthread.c:341) [ 930.881964] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 930.881968] | [ 973.248609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 973.248947] rcu: 3-....: (961908 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=220534 | [ 973.249461] rcu: (t=962279 jiffies g=7477 q=1771 ncpus=4) [ 973.250227] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 973.250824] Workqueue: netns cleanup_net [ 973.251038] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 973.251330] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 973.252298] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 973.252574] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 973.252951] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 973.253325] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 973.253700] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 973.254072] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 973.254448] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 973.254872] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 973.255174] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 973.255550] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 973.255925] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 973.256299] PKRU: 55555554 [ 973.256447] Call Trace: [ 973.256584] [ 973.256700] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 973.256938] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 973.257179] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 973.257392] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 973.257599] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 973.257807] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 973.258047] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 973.258279] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 973.258562] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 973.258781] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 973.259036] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 973.259276] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 973.259506] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 973.259783] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 973.260048] [ 973.260164] [ 973.260282] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 973.260565] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 973.260827] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 973.261036] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 973.261238] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 973.261504] ? up_read (kernel/locking/rwsem.c:1622) [ 973.261683] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 973.261923] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 973.262193] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 973.262449] __tcf_action_put (net/sched/act_api.c:377) [ 973.262659] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 973.262875] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 973.263081] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 973.263363] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 973.263618] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 973.263865] tcf_proto_destroy (net/sched/cls_api.c:418) [ 973.264076] tcf_chain_flush (net/sched/cls_api.c:737) [ 973.264282] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 973.264488] tcf_block_put (net/sched/cls_api.c:1543) [ 973.264683] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 973.264924] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 973.265132] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 973.265327] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 973.265617] default_device_exit_batch (net/core/dev.c:11592) [ 973.265874] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 973.266069] ? process_one_work (kernel/workqueue.c:2606) [ 973.266293] process_one_work (kernel/workqueue.c:2633) [ 973.266508] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 973.266714] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 973.266947] kthread (kernel/kthread.c:388) [ 973.267121] ? __pfx_kthread (kernel/kthread.c:341) [ 973.267323] ret_from_fork (arch/x86/kernel/process.c:147) [ 973.267519] ? __pfx_kthread (kernel/kthread.c:341) [ 973.267722] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1029.184622] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 1018185 jiffies s: 4157 root: 0x8/. | [ 1029.185267] rcu: blocking rcu_node structures (internal RCU debug): | [ 1029.185624] Sending NMI from CPU 2 to CPUs 3: | [ 1029.185872] NMI backtrace for cpu 3 [ 1029.185878] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1029.185880] Workqueue: netns cleanup_net [ 1029.185885] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1029.185889] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1029.185890] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1029.185891] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1029.185892] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1029.185893] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1029.185894] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1029.185895] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1029.185898] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1029.185899] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1029.185900] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1029.185900] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1029.185901] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1029.185901] PKRU: 55555554 [ 1029.185902] Call Trace: [ 1029.185904] [ 1029.185905] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 1029.185909] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 1029.185913] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 1029.185915] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 1029.185918] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 1029.185920] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 1029.185923] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1029.185926] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1029.185928] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1029.185930] [ 1029.185930] [ 1029.185931] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1029.185934] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1029.185936] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1029.185938] ? up_read (kernel/locking/rwsem.c:1622) [ 1029.185941] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1029.185945] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1029.185946] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1029.185948] __tcf_action_put (net/sched/act_api.c:377) [ 1029.185953] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1029.185955] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1029.185957] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1029.185961] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1029.185967] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1029.185972] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1029.185974] tcf_chain_flush (net/sched/cls_api.c:737) [ 1029.185976] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1029.185977] tcf_block_put (net/sched/cls_api.c:1543) [ 1029.185979] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1029.185983] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1029.185986] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1029.185988] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1029.185991] default_device_exit_batch (net/core/dev.c:11592) [ 1029.185994] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1029.185996] ? process_one_work (kernel/workqueue.c:2606) [ 1029.185999] process_one_work (kernel/workqueue.c:2633) [ 1029.186002] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1029.186004] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1029.186006] kthread (kernel/kthread.c:388) [ 1029.186007] ? __pfx_kthread (kernel/kthread.c:341) [ 1029.186009] ret_from_fork (arch/x86/kernel/process.c:147) [ 1029.186012] ? __pfx_kthread (kernel/kthread.c:341) [ 1029.186014] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1029.186017] | [ 1051.271609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1051.271946] rcu: 3-....: (1039898 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=238415 | [ 1051.272461] rcu: (t=1040302 jiffies g=7477 q=1773 ncpus=4) [ 1051.273207] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1051.273802] Workqueue: netns cleanup_net [ 1051.274015] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1051.274308] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1051.275273] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1051.275549] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1051.275925] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1051.276298] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1051.276675] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1051.277045] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1051.277420] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1051.277847] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1051.278149] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1051.278524] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1051.278899] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1051.279272] PKRU: 55555554 [ 1051.279419] Call Trace: [ 1051.279555] [ 1051.279672] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1051.279911] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1051.280156] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 1051.280369] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 1051.280578] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 1051.280785] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1051.281025] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1051.281257] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1051.281541] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1051.281770] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1051.282026] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1051.282266] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1051.282495] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1051.282771] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1051.283035] [ 1051.283157] [ 1051.283276] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1051.283557] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1051.283819] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1051.284027] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1051.284225] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1051.284490] ? up_read (kernel/locking/rwsem.c:1622) [ 1051.284670] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1051.284909] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1051.285172] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1051.285427] __tcf_action_put (net/sched/act_api.c:377) [ 1051.285637] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1051.285853] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1051.286060] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1051.286354] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1051.286609] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1051.286857] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1051.287069] tcf_chain_flush (net/sched/cls_api.c:737) [ 1051.287275] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1051.287482] tcf_block_put (net/sched/cls_api.c:1543) [ 1051.287678] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1051.287919] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1051.288126] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1051.288321] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1051.288610] default_device_exit_batch (net/core/dev.c:11592) [ 1051.288869] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1051.289064] ? process_one_work (kernel/workqueue.c:2606) [ 1051.289289] process_one_work (kernel/workqueue.c:2633) [ 1051.289504] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1051.289710] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1051.289940] kthread (kernel/kthread.c:388) [ 1051.290113] ? __pfx_kthread (kernel/kthread.c:341) [ 1051.290316] ret_from_fork (arch/x86/kernel/process.c:147) [ 1051.290512] ? __pfx_kthread (kernel/kthread.c:341) [ 1051.290714] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1127.488624] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 1116489 jiffies s: 4157 root: 0x8/. | [ 1127.489238] rcu: blocking rcu_node structures (internal RCU debug): | [ 1127.489573] Sending NMI from CPU 2 to CPUs 3: | [ 1127.489824] NMI backtrace for cpu 3 [ 1127.489828] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1127.489829] Workqueue: netns cleanup_net [ 1127.489834] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:13 ./arch/x86/include/asm/vdso/processor.h:18 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1127.489838] Code: 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc 90 eb e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba All code ======== 0: 55 push %rbp 1: 41 54 push %r12 3: 55 push %rbp 4: 48 89 fd mov %rdi,%rbp 7: 53 push %rbx 8: 66 90 xchg %ax,%ax a: ba 01 00 00 00 mov $0x1,%edx f: 8b 45 00 mov 0x0(%rbp),%eax 12: 85 c0 test %eax,%eax 14: 75 14 jne 0x2a 16: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 1b: 85 c0 test %eax,%eax 1d: 75 f0 jne 0xf 1f: 5b pop %rbx 20: 5d pop %rbp 21: 41 5c pop %r12 23: 41 5d pop %r13 25: c3 ret 26: cc int3 27: cc int3 28: cc int3 29: cc int3 2a:* f3 90 pause <-- trapping instruction 2c: eb e1 jmp 0xf 2e: 81 fe 00 01 00 00 cmp $0x100,%esi 34: 74 52 je 0x88 36: 40 30 f6 xor %sil,%sil 39: 85 f6 test %esi,%esi 3b: 75 75 jne 0xb2 3d: f0 lock 3e: 0f .byte 0xf 3f: ba .byte 0xba Code starting with the faulting instruction =========================================== 0: f3 90 pause 2: eb e1 jmp 0xffffffffffffffe5 4: 81 fe 00 01 00 00 cmp $0x100,%esi a: 74 52 je 0x5e c: 40 30 f6 xor %sil,%sil f: 85 f6 test %esi,%esi 11: 75 75 jne 0x88 13: f0 lock 14: 0f .byte 0xf 15: ba .byte 0xba [ 1127.489839] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1127.489840] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1127.489841] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1127.489842] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1127.489843] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1127.489844] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1127.489846] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1127.489847] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1127.489848] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1127.489848] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1127.489849] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1127.489850] PKRU: 55555554 [ 1127.489850] Call Trace: [ 1127.489852] [ 1127.489853] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 1127.489857] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 1127.489861] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 1127.489863] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 1127.489866] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 1127.489868] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 1127.489871] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:13 ./arch/x86/include/asm/vdso/processor.h:18 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1127.489873] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:13 ./arch/x86/include/asm/vdso/processor.h:18 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1127.489876] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:13 ./arch/x86/include/asm/vdso/processor.h:18 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1127.489878] [ 1127.489878] [ 1127.489879] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1127.489883] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1127.489884] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1127.489887] ? up_read (kernel/locking/rwsem.c:1622) [ 1127.489890] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1127.489893] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1127.489895] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1127.489897] __tcf_action_put (net/sched/act_api.c:377) [ 1127.489900] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1127.489902] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1127.489904] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1127.489910] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1127.489915] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1127.489920] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1127.489922] tcf_chain_flush (net/sched/cls_api.c:737) [ 1127.489923] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1127.489925] tcf_block_put (net/sched/cls_api.c:1543) [ 1127.489927] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1127.489931] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1127.489934] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1127.489936] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1127.489939] default_device_exit_batch (net/core/dev.c:11592) [ 1127.489942] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1127.489944] ? process_one_work (kernel/workqueue.c:2606) [ 1127.489947] process_one_work (kernel/workqueue.c:2633) [ 1127.489950] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1127.489952] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1127.489954] kthread (kernel/kthread.c:388) [ 1127.489956] ? __pfx_kthread (kernel/kthread.c:341) [ 1127.489957] ret_from_fork (arch/x86/kernel/process.c:147) [ 1127.489960] ? __pfx_kthread (kernel/kthread.c:341) [ 1127.489961] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1127.489965] | [ 1129.294608] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1129.294931] rcu: 3-....: (1117888 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=256481 | [ 1129.295447] rcu: (t=1118325 jiffies g=7477 q=1773 ncpus=4) [ 1129.296191] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1129.296785] Workqueue: netns cleanup_net [ 1129.296999] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1129.297289] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1129.298251] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1129.298527] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1129.298901] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1129.299274] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1129.299648] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1129.300022] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1129.300397] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1129.300819] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1129.301123] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1129.301504] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1129.301879] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1129.302251] PKRU: 55555554 [ 1129.302400] Call Trace: [ 1129.302535] [ 1129.302650] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1129.302885] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1129.303129] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 1129.303340] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 1129.303546] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 1129.303755] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1129.303992] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1129.304227] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1129.304508] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1129.304728] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1129.304983] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1129.305222] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1129.305452] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1129.305729] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1129.305997] [ 1129.306115] [ 1129.306233] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1129.306515] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1129.306776] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1129.306982] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1129.307179] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1129.307449] ? up_read (kernel/locking/rwsem.c:1622) [ 1129.307626] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1129.307868] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1129.308132] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1129.308387] __tcf_action_put (net/sched/act_api.c:377) [ 1129.308594] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1129.308809] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1129.309018] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1129.309299] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1129.309555] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1129.309800] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1129.310013] tcf_chain_flush (net/sched/cls_api.c:737) [ 1129.310218] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1129.310424] tcf_block_put (net/sched/cls_api.c:1543) [ 1129.310618] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1129.310860] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1129.311069] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1129.311264] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1129.311558] default_device_exit_batch (net/core/dev.c:11592) [ 1129.311815] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1129.312009] ? process_one_work (kernel/workqueue.c:2606) [ 1129.312235] process_one_work (kernel/workqueue.c:2633) [ 1129.312450] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1129.312653] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1129.312883] kthread (kernel/kthread.c:388) [ 1129.313054] ? __pfx_kthread (kernel/kthread.c:341) [ 1129.313258] ret_from_fork (arch/x86/kernel/process.c:147) [ 1129.313452] ? __pfx_kthread (kernel/kthread.c:341) [ 1129.313655] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1129.313868] | [ 1207.316609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1207.316949] rcu: 3-....: (1195892 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=274564 | [ 1207.317467] rcu: (t=1196347 jiffies g=7477 q=1773 ncpus=4) [ 1207.318210] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1207.318806] Workqueue: netns cleanup_net [ 1207.319022] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1207.319319] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1207.320284] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1207.320558] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1207.320938] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1207.321312] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1207.321689] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1207.322064] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1207.322440] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1207.322864] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1207.323171] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1207.323545] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1207.323920] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1207.324291] PKRU: 55555554 [ 1207.324441] Call Trace: [ 1207.324577] [ 1207.324694] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1207.324929] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1207.325172] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 1207.325383] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 1207.325593] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 1207.325802] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1207.326039] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1207.326276] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1207.326559] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1207.326779] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1207.327034] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1207.327273] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1207.327505] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1207.327784] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1207.328048] [ 1207.328166] [ 1207.328284] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1207.328567] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1207.328830] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1207.329038] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1207.329236] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1207.329502] ? up_read (kernel/locking/rwsem.c:1622) [ 1207.329680] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1207.329921] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1207.330185] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1207.330441] __tcf_action_put (net/sched/act_api.c:377) [ 1207.330649] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1207.330863] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1207.331078] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1207.331360] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1207.331616] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1207.331861] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1207.332074] tcf_chain_flush (net/sched/cls_api.c:737) [ 1207.332280] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1207.332488] tcf_block_put (net/sched/cls_api.c:1543) [ 1207.332683] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1207.332924] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1207.333132] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1207.333327] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1207.333613] default_device_exit_batch (net/core/dev.c:11592) [ 1207.333868] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1207.334063] ? process_one_work (kernel/workqueue.c:2606) [ 1207.334293] process_one_work (kernel/workqueue.c:2633) [ 1207.334509] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1207.334713] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1207.334944] kthread (kernel/kthread.c:388) [ 1207.335115] ? __pfx_kthread (kernel/kthread.c:341) [ 1207.335318] ret_from_fork (arch/x86/kernel/process.c:147) [ 1207.335514] ? __pfx_kthread (kernel/kthread.c:341) [ 1207.335722] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1225.792621] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 1214793 jiffies s: 4157 root: 0x8/. | [ 1225.793241] rcu: blocking rcu_node structures (internal RCU debug): | [ 1225.793572] Sending NMI from CPU 2 to CPUs 3: | [ 1225.793824] NMI backtrace for cpu 3 [ 1225.793830] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1225.793831] Workqueue: netns cleanup_net [ 1225.793834] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1225.793837] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1225.793839] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1225.793840] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1225.793840] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1225.793841] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1225.793842] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1225.793842] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1225.793845] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1225.793846] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1225.793847] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1225.793848] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1225.793848] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1225.793849] PKRU: 55555554 [ 1225.793849] Call Trace: [ 1225.793850] [ 1225.793851] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 1225.793855] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 1225.793858] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 1225.793860] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 1225.793864] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 1225.793866] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 1225.793869] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1225.793872] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1225.793874] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1225.793876] [ 1225.793876] [ 1225.793877] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1225.793880] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1225.793881] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1225.793884] ? up_read (kernel/locking/rwsem.c:1622) [ 1225.793886] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1225.793889] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1225.793891] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1225.793893] __tcf_action_put (net/sched/act_api.c:377) [ 1225.793895] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1225.793897] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1225.793899] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1225.793905] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1225.793910] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1225.793915] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1225.793917] tcf_chain_flush (net/sched/cls_api.c:737) [ 1225.793919] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1225.793920] tcf_block_put (net/sched/cls_api.c:1543) [ 1225.793922] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1225.793926] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1225.793928] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1225.793931] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1225.793934] default_device_exit_batch (net/core/dev.c:11592) [ 1225.793936] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1225.793938] ? process_one_work (kernel/workqueue.c:2606) [ 1225.793941] process_one_work (kernel/workqueue.c:2633) [ 1225.793944] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1225.793946] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1225.793948] kthread (kernel/kthread.c:388) [ 1225.793950] ? __pfx_kthread (kernel/kthread.c:341) [ 1225.793951] ret_from_fork (arch/x86/kernel/process.c:147) [ 1225.793953] ? __pfx_kthread (kernel/kthread.c:341) [ 1225.793955] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1225.793958] | [ 1285.339609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1285.339950] rcu: 3-....: (1273882 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=292366 | [ 1285.340467] rcu: (t=1274370 jiffies g=7477 q=1773 ncpus=4) [ 1285.341219] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1285.341816] Workqueue: netns cleanup_net [ 1285.342031] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1285.342325] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1285.343290] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1285.343567] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1285.343942] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1285.344317] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1285.344694] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1285.345070] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1285.345450] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1285.345875] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1285.346181] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1285.346556] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1285.346930] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1285.347305] PKRU: 55555554 [ 1285.347453] Call Trace: [ 1285.347591] [ 1285.347706] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1285.347945] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1285.348189] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 1285.348402] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 1285.348610] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 1285.348817] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1285.349058] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1285.349294] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1285.349577] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1285.349796] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1285.350053] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1285.350294] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1285.350530] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1285.350811] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1285.351075] [ 1285.351194] [ 1285.351312] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1285.351600] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1285.351864] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1285.352071] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1285.352269] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1285.352534] ? up_read (kernel/locking/rwsem.c:1622) [ 1285.352715] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1285.352956] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1285.353222] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1285.353480] __tcf_action_put (net/sched/act_api.c:377) [ 1285.353691] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1285.353906] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1285.354113] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1285.354397] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1285.354653] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1285.354899] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1285.355111] tcf_chain_flush (net/sched/cls_api.c:737) [ 1285.355319] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1285.355526] tcf_block_put (net/sched/cls_api.c:1543) [ 1285.355722] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1285.355964] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1285.356174] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1285.356369] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1285.356658] default_device_exit_batch (net/core/dev.c:11592) [ 1285.356916] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1285.357114] ? process_one_work (kernel/workqueue.c:2606) [ 1285.357340] process_one_work (kernel/workqueue.c:2633) [ 1285.357560] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1285.357764] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1285.357995] kthread (kernel/kthread.c:388) [ 1285.358168] ? __pfx_kthread (kernel/kthread.c:341) [ 1285.358370] ret_from_fork (arch/x86/kernel/process.c:147) [ 1285.358567] ? __pfx_kthread (kernel/kthread.c:341) [ 1285.358775] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1324.096626] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 1313097 jiffies s: 4157 root: 0x8/. | [ 1324.097244] rcu: blocking rcu_node structures (internal RCU debug): | [ 1324.097583] Sending NMI from CPU 2 to CPUs 3: | [ 1324.097836] NMI backtrace for cpu 3 [ 1324.097842] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1324.097844] Workqueue: netns cleanup_net [ 1324.097848] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1324.097852] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1324.097854] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1324.097855] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1324.097856] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1324.097857] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1324.097858] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1324.097858] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1324.097861] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1324.097862] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1324.097863] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1324.097864] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1324.097864] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1324.097865] PKRU: 55555554 [ 1324.097865] Call Trace: [ 1324.097867] [ 1324.097868] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 1324.097872] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 1324.097875] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 1324.097878] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 1324.097881] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 1324.097883] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 1324.097887] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1324.097889] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1324.097892] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1324.097894] [ 1324.097894] [ 1324.097895] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1324.097898] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1324.097899] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1324.097902] ? up_read (kernel/locking/rwsem.c:1622) [ 1324.097904] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1324.097908] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1324.097910] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1324.097912] __tcf_action_put (net/sched/act_api.c:377) [ 1324.097915] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1324.097917] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1324.097919] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1324.097924] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1324.097930] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1324.097934] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1324.097937] tcf_chain_flush (net/sched/cls_api.c:737) [ 1324.097938] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1324.097940] tcf_block_put (net/sched/cls_api.c:1543) [ 1324.097942] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1324.097946] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1324.097949] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1324.097951] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1324.097954] default_device_exit_batch (net/core/dev.c:11592) [ 1324.097957] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1324.097959] ? process_one_work (kernel/workqueue.c:2606) [ 1324.097961] process_one_work (kernel/workqueue.c:2633) [ 1324.097964] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1324.097966] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1324.097968] kthread (kernel/kthread.c:388) [ 1324.097969] ? __pfx_kthread (kernel/kthread.c:341) [ 1324.097971] ret_from_fork (arch/x86/kernel/process.c:147) [ 1324.097974] ? __pfx_kthread (kernel/kthread.c:341) [ 1324.097976] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1324.097979] | [ 1363.362609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1363.362944] rcu: 3-....: (1351872 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=310376 | [ 1363.363460] rcu: (t=1352393 jiffies g=7477 q=1773 ncpus=4) [ 1363.364201] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1363.364792] Workqueue: netns cleanup_net [ 1363.365005] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 1363.365298] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 <85> c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: f3 0f 1e fa endbr64 12: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 17: 41 55 push %r13 19: 41 54 push %r12 1b: 55 push %rbp 1c: 48 89 fd mov %rdi,%rbp 1f: 53 push %rbx 20: 66 90 xchg %ax,%ax 22: ba 01 00 00 00 mov $0x1,%edx 27: 8b 45 00 mov 0x0(%rbp),%eax 2a:* 85 c0 test %eax,%eax <-- trapping instruction 2c: 75 14 jne 0x42 2e: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 33: 85 c0 test %eax,%eax 35: 75 f0 jne 0x27 37: 5b pop %rbx 38: 5d pop %rbp 39: 41 5c pop %r12 3b: 41 5d pop %r13 3d: c3 ret 3e: cc int3 3f: cc int3 Code starting with the faulting instruction =========================================== 0: 85 c0 test %eax,%eax 2: 75 14 jne 0x18 4: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 9: 85 c0 test %eax,%eax b: 75 f0 jne 0xfffffffffffffffd d: 5b pop %rbx e: 5d pop %rbp f: 41 5c pop %r12 11: 41 5d pop %r13 13: c3 ret 14: cc int3 15: cc int3 [ 1363.366264] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1363.366540] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1363.366913] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1363.367287] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1363.367660] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1363.368032] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1363.368407] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1363.368830] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1363.369133] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1363.369505] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1363.369879] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1363.370256] PKRU: 55555554 [ 1363.370403] Call Trace: [ 1363.370539] [ 1363.370654] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1363.370892] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1363.371133] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 1363.371347] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 1363.371553] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 1363.371759] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1363.371999] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1363.372231] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1363.372514] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1363.372732] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1363.372988] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1363.373227] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1363.373457] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1363.373733] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1363.373997] [ 1363.374113] [ 1363.374231] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1363.374515] ? queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 1363.374775] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1363.374983] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1363.375180] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1363.375445] ? up_read (kernel/locking/rwsem.c:1622) [ 1363.375625] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1363.375862] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1363.376125] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1363.376379] __tcf_action_put (net/sched/act_api.c:377) [ 1363.376587] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1363.376803] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1363.377009] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1363.377289] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1363.377543] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1363.377790] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1363.378001] tcf_chain_flush (net/sched/cls_api.c:737) [ 1363.378207] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1363.378414] tcf_block_put (net/sched/cls_api.c:1543) [ 1363.378608] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1363.378848] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1363.379056] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1363.379250] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1363.379538] default_device_exit_batch (net/core/dev.c:11592) [ 1363.379795] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1363.379990] ? process_one_work (kernel/workqueue.c:2606) [ 1363.380224] process_one_work (kernel/workqueue.c:2633) [ 1363.380439] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1363.380643] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1363.380871] kthread (kernel/kthread.c:388) [ 1363.381044] ? __pfx_kthread (kernel/kthread.c:341) [ 1363.381246] ret_from_fork (arch/x86/kernel/process.c:147) [ 1363.381442] ? __pfx_kthread (kernel/kthread.c:341) [ 1363.381647] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1422.400620] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 1411401 jiffies s: 4157 root: 0x8/. | [ 1422.401285] rcu: blocking rcu_node structures (internal RCU debug): | [ 1422.401636] Sending NMI from CPU 2 to CPUs 3: | [ 1422.401886] NMI backtrace for cpu 3 [ 1422.401891] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1422.401892] Workqueue: netns cleanup_net [ 1422.401897] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1422.401901] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1422.401902] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1422.401903] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1422.401904] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1422.401905] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1422.401906] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1422.401907] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1422.401909] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1422.401910] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1422.401911] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1422.401912] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1422.401912] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1422.401913] PKRU: 55555554 [ 1422.401913] Call Trace: [ 1422.401915] [ 1422.401917] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 1422.401920] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 1422.401924] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 1422.401926] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 1422.401929] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 1422.401931] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 1422.401934] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1422.401937] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1422.401939] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1422.401941] [ 1422.401941] [ 1422.401942] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1422.401946] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1422.401948] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1422.401951] ? up_read (kernel/locking/rwsem.c:1622) [ 1422.401954] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1422.401957] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1422.401959] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1422.401961] __tcf_action_put (net/sched/act_api.c:377) [ 1422.401964] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1422.401966] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1422.401968] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1422.401974] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1422.401979] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1422.401984] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1422.401987] tcf_chain_flush (net/sched/cls_api.c:737) [ 1422.401988] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1422.401990] tcf_block_put (net/sched/cls_api.c:1543) [ 1422.401992] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1422.401997] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1422.402000] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1422.402002] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1422.402005] default_device_exit_batch (net/core/dev.c:11592) [ 1422.402008] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1422.402010] ? process_one_work (kernel/workqueue.c:2606) [ 1422.402013] process_one_work (kernel/workqueue.c:2633) [ 1422.402016] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1422.402019] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1422.402020] kthread (kernel/kthread.c:388) [ 1422.402022] ? __pfx_kthread (kernel/kthread.c:341) [ 1422.402024] ret_from_fork (arch/x86/kernel/process.c:147) [ 1422.402027] ? __pfx_kthread (kernel/kthread.c:341) [ 1422.402028] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1422.402032] | [ 1441.385608] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1441.385937] rcu: 3-....: (1429862 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=328196 | [ 1441.386453] rcu: (t=1430416 jiffies g=7477 q=1773 ncpus=4) [ 1441.387192] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1441.387784] Workqueue: netns cleanup_net [ 1441.387995] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:13 ./arch/x86/include/asm/vdso/processor.h:18 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1441.388288] Code: 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc 90 eb e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba All code ======== 0: 55 push %rbp 1: 41 54 push %r12 3: 55 push %rbp 4: 48 89 fd mov %rdi,%rbp 7: 53 push %rbx 8: 66 90 xchg %ax,%ax a: ba 01 00 00 00 mov $0x1,%edx f: 8b 45 00 mov 0x0(%rbp),%eax 12: 85 c0 test %eax,%eax 14: 75 14 jne 0x2a 16: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 1b: 85 c0 test %eax,%eax 1d: 75 f0 jne 0xf 1f: 5b pop %rbx 20: 5d pop %rbp 21: 41 5c pop %r12 23: 41 5d pop %r13 25: c3 ret 26: cc int3 27: cc int3 28: cc int3 29: cc int3 2a:* f3 90 pause <-- trapping instruction 2c: eb e1 jmp 0xf 2e: 81 fe 00 01 00 00 cmp $0x100,%esi 34: 74 52 je 0x88 36: 40 30 f6 xor %sil,%sil 39: 85 f6 test %esi,%esi 3b: 75 75 jne 0xb2 3d: f0 lock 3e: 0f .byte 0xf 3f: ba .byte 0xba Code starting with the faulting instruction =========================================== 0: f3 90 pause 2: eb e1 jmp 0xffffffffffffffe5 4: 81 fe 00 01 00 00 cmp $0x100,%esi a: 74 52 je 0x5e c: 40 30 f6 xor %sil,%sil f: 85 f6 test %esi,%esi 11: 75 75 jne 0x88 13: f0 lock 14: 0f .byte 0xf 15: ba .byte 0xba [ 1441.389242] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1441.389520] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1441.389898] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1441.390273] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1441.390644] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1441.391017] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1441.391389] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1441.391812] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1441.392116] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1441.392489] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1441.392863] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1441.393236] PKRU: 55555554 [ 1441.393383] Call Trace: [ 1441.393518] [ 1441.393634] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1441.393871] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1441.394118] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 1441.394329] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 1441.394536] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 1441.394742] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1441.394981] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1441.395215] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1441.395496] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1441.395714] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1441.395968] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1441.396208] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1441.396436] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1441.396714] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1441.396977] [ 1441.397094] [ 1441.397211] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1441.397494] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:13 ./arch/x86/include/asm/vdso/processor.h:18 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1441.397754] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1441.397961] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1441.398158] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1441.398422] ? up_read (kernel/locking/rwsem.c:1622) [ 1441.398601] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1441.398840] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1441.399104] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1441.399358] __tcf_action_put (net/sched/act_api.c:377) [ 1441.399566] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1441.399788] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1441.399994] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1441.400273] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1441.400527] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1441.400775] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1441.400988] tcf_chain_flush (net/sched/cls_api.c:737) [ 1441.401196] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1441.401403] tcf_block_put (net/sched/cls_api.c:1543) [ 1441.401596] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1441.401836] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1441.402047] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1441.402242] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1441.402527] default_device_exit_batch (net/core/dev.c:11592) [ 1441.402784] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1441.402979] ? process_one_work (kernel/workqueue.c:2606) [ 1441.403203] process_one_work (kernel/workqueue.c:2633) [ 1441.403422] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1441.403625] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1441.403852] kthread (kernel/kthread.c:388) [ 1441.404025] ? __pfx_kthread (kernel/kthread.c:341) [ 1441.404227] ret_from_fork (arch/x86/kernel/process.c:147) [ 1441.404421] ? __pfx_kthread (kernel/kthread.c:341) [ 1441.404622] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1441.404837] | [ 1519.407609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1519.407945] rcu: 3-....: (1507866 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=345826 | [ 1519.408460] rcu: (t=1508438 jiffies g=7477 q=1773 ncpus=4) [ 1519.409212] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1519.409815] Workqueue: netns cleanup_net [ 1519.410028] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1519.410321] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1519.411284] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1519.411559] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1519.411935] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1519.412313] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1519.412687] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1519.413059] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1519.413434] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1519.413858] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1519.414160] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1519.414536] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1519.414910] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1519.415284] PKRU: 55555554 [ 1519.415435] Call Trace: [ 1519.415571] [ 1519.415688] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1519.415925] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1519.416167] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 1519.416379] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 1519.416588] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 1519.416797] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1519.417036] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1519.417269] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1519.417549] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1519.417770] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1519.418027] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1519.418266] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1519.418496] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1519.418774] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1519.419037] [ 1519.419155] [ 1519.419274] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1519.419563] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1519.419825] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1519.420033] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1519.420234] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1519.420499] ? up_read (kernel/locking/rwsem.c:1622) [ 1519.420676] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1519.420915] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1519.421177] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1519.421432] __tcf_action_put (net/sched/act_api.c:377) [ 1519.421640] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1519.421857] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1519.422064] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1519.422343] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1519.422598] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1519.422846] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1519.423056] tcf_chain_flush (net/sched/cls_api.c:737) [ 1519.423262] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1519.423471] tcf_block_put (net/sched/cls_api.c:1543) [ 1519.423665] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1519.423906] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1519.424115] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1519.424310] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1519.424598] default_device_exit_batch (net/core/dev.c:11592) [ 1519.424855] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1519.425049] ? process_one_work (kernel/workqueue.c:2606) [ 1519.425274] process_one_work (kernel/workqueue.c:2633) [ 1519.425490] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1519.425695] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1519.425924] kthread (kernel/kthread.c:388) [ 1519.426098] ? __pfx_kthread (kernel/kthread.c:341) [ 1519.426301] ret_from_fork (arch/x86/kernel/process.c:147) [ 1519.426497] ? __pfx_kthread (kernel/kthread.c:341) [ 1519.426703] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1520.704622] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 1509705 jiffies s: 4157 root: 0x8/. | [ 1520.705228] rcu: blocking rcu_node structures (internal RCU debug): | [ 1520.705558] Sending NMI from CPU 2 to CPUs 3: | [ 1520.705812] NMI backtrace for cpu 3 [ 1520.705816] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1520.705817] Workqueue: netns cleanup_net [ 1520.705819] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1520.705821] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1520.705822] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1520.705823] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1520.705824] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1520.705825] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1520.705825] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1520.705826] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1520.705828] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1520.705829] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1520.705830] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1520.705831] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1520.705831] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1520.705832] PKRU: 55555554 [ 1520.705832] Call Trace: [ 1520.705833] [ 1520.705834] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 1520.705838] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 1520.705841] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 1520.705844] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 1520.705847] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 1520.705849] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 1520.705852] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1520.705854] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1520.705857] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1520.705859] [ 1520.705859] [ 1520.705860] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1520.705862] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1520.705863] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1520.705865] ? up_read (kernel/locking/rwsem.c:1622) [ 1520.705867] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1520.705870] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1520.705871] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1520.705873] __tcf_action_put (net/sched/act_api.c:377) [ 1520.705875] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1520.705876] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1520.705878] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1520.705883] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1520.705888] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1520.705892] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1520.705895] tcf_chain_flush (net/sched/cls_api.c:737) [ 1520.705896] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1520.705898] tcf_block_put (net/sched/cls_api.c:1543) [ 1520.705899] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1520.705903] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1520.705905] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1520.705907] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1520.705909] default_device_exit_batch (net/core/dev.c:11592) [ 1520.705911] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1520.705914] ? process_one_work (kernel/workqueue.c:2606) [ 1520.705915] process_one_work (kernel/workqueue.c:2633) [ 1520.705918] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1520.705920] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1520.705921] kthread (kernel/kthread.c:388) [ 1520.705923] ? __pfx_kthread (kernel/kthread.c:341) [ 1520.705924] ret_from_fork (arch/x86/kernel/process.c:147) [ 1520.705926] ? __pfx_kthread (kernel/kthread.c:341) [ 1520.705927] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1520.705931] | [ 1597.429610] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1597.429949] rcu: 3-....: (1585855 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=364620 | [ 1597.430460] rcu: (t=1586460 jiffies g=7477 q=1773 ncpus=4) [ 1597.431199] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1597.431787] Workqueue: netns cleanup_net [ 1597.431998] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1597.432289] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1597.433243] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1597.433517] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1597.433889] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1597.434260] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1597.434632] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1597.435003] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1597.435373] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1597.435795] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1597.436096] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1597.436467] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1597.436839] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1597.437210] PKRU: 55555554 [ 1597.437356] Call Trace: [ 1597.437491] [ 1597.437605] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1597.437841] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1597.438082] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 1597.438293] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 1597.438498] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 1597.438703] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1597.438946] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1597.439177] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1597.439458] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1597.439675] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1597.439929] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1597.440167] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1597.440394] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1597.440669] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1597.440932] [ 1597.441048] [ 1597.441164] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1597.441446] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1597.441707] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1597.441912] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1597.442109] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1597.442373] ? up_read (kernel/locking/rwsem.c:1622) [ 1597.442552] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1597.442789] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1597.443051] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1597.443302] __tcf_action_put (net/sched/act_api.c:377) [ 1597.443511] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1597.443725] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1597.443930] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1597.444208] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1597.444461] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1597.444706] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1597.444916] tcf_chain_flush (net/sched/cls_api.c:737) [ 1597.445123] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1597.445330] tcf_block_put (net/sched/cls_api.c:1543) [ 1597.445524] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1597.445761] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1597.445969] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1597.446162] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1597.446448] default_device_exit_batch (net/core/dev.c:11592) [ 1597.446704] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1597.446897] ? process_one_work (kernel/workqueue.c:2606) [ 1597.447118] process_one_work (kernel/workqueue.c:2633) [ 1597.447333] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1597.447534] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1597.447763] kthread (kernel/kthread.c:388) [ 1597.447935] ? __pfx_kthread (kernel/kthread.c:341) [ 1597.448139] ret_from_fork (arch/x86/kernel/process.c:147) [ 1597.448332] ? __pfx_kthread (kernel/kthread.c:341) [ 1597.448532] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1619.008623] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 1608009 jiffies s: 4157 root: 0x8/. | [ 1619.009236] rcu: blocking rcu_node structures (internal RCU debug): | [ 1619.009572] Sending NMI from CPU 2 to CPUs 3: | [ 1619.009826] NMI backtrace for cpu 3 [ 1619.009830] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1619.009832] Workqueue: netns cleanup_net [ 1619.009836] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1619.009840] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1619.009841] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1619.009843] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1619.009844] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1619.009845] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1619.009846] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1619.009846] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1619.009849] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1619.009850] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1619.009851] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1619.009851] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1619.009852] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1619.009853] PKRU: 55555554 [ 1619.009853] Call Trace: [ 1619.009855] [ 1619.009856] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 1619.009860] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 1619.009864] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 1619.009866] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 1619.009869] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 1619.009871] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 1619.009875] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1619.009877] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1619.009879] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1619.009882] [ 1619.009882] [ 1619.009883] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1619.009886] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1619.009887] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1619.009890] ? up_read (kernel/locking/rwsem.c:1622) [ 1619.009892] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1619.009895] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1619.009897] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1619.009899] __tcf_action_put (net/sched/act_api.c:377) [ 1619.009902] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1619.009904] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1619.009906] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1619.009911] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1619.009917] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1619.009922] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1619.009924] tcf_chain_flush (net/sched/cls_api.c:737) [ 1619.009925] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1619.009927] tcf_block_put (net/sched/cls_api.c:1543) [ 1619.009929] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1619.009933] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1619.009936] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1619.009938] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1619.009941] default_device_exit_batch (net/core/dev.c:11592) [ 1619.009943] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1619.009945] ? process_one_work (kernel/workqueue.c:2606) [ 1619.009948] process_one_work (kernel/workqueue.c:2633) [ 1619.009951] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1619.009953] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1619.009955] kthread (kernel/kthread.c:388) [ 1619.009956] ? __pfx_kthread (kernel/kthread.c:341) [ 1619.009958] ret_from_fork (arch/x86/kernel/process.c:147) [ 1619.009961] ? __pfx_kthread (kernel/kthread.c:341) [ 1619.009962] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1619.009966] | [ 1675.452609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1675.452961] rcu: 3-....: (1663845 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=382815 | [ 1675.453488] rcu: (t=1664483 jiffies g=7477 q=1773 ncpus=4) [ 1675.454242] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1675.454851] Workqueue: netns cleanup_net [ 1675.455071] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 1675.455371] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 <8b> 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: 90 nop f: 90 nop 10: 90 nop 11: f3 0f 1e fa endbr64 15: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 1a: 41 55 push %r13 1c: 41 54 push %r12 1e: 55 push %rbp 1f: 48 89 fd mov %rdi,%rbp 22: 53 push %rbx 23: 66 90 xchg %ax,%ax 25: ba 01 00 00 00 mov $0x1,%edx 2a:* 8b 45 00 mov 0x0(%rbp),%eax <-- trapping instruction 2d: 85 c0 test %eax,%eax 2f: 75 14 jne 0x45 31: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 36: 85 c0 test %eax,%eax 38: 75 f0 jne 0x2a 3a: 5b pop %rbx 3b: 5d pop %rbp 3c: 41 5c pop %r12 3e: 41 5d pop %r13 Code starting with the faulting instruction =========================================== 0: 8b 45 00 mov 0x0(%rbp),%eax 3: 85 c0 test %eax,%eax 5: 75 14 jne 0x1b 7: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) c: 85 c0 test %eax,%eax e: 75 f0 jne 0x0 10: 5b pop %rbx 11: 5d pop %rbp 12: 41 5c pop %r12 14: 41 5d pop %r13 [ 1675.456350] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1675.456641] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1675.457014] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1675.457395] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1675.457779] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1675.458154] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1675.458542] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1675.458974] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1675.459279] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1675.459667] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1675.460041] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1675.460423] PKRU: 55555554 [ 1675.460569] Call Trace: [ 1675.460715] [ 1675.460830] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1675.461073] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1675.461317] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 1675.461536] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 1675.461755] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 1675.461962] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1675.462202] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1675.462445] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1675.462737] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1675.462957] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1675.463214] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1675.463461] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1675.463699] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1675.463979] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1675.464245] [ 1675.464365] [ 1675.464487] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1675.464782] ? queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 1675.465044] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1675.465254] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1675.465460] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1675.465736] ? up_read (kernel/locking/rwsem.c:1622) [ 1675.465915] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1675.466158] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1675.466429] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1675.466693] __tcf_action_put (net/sched/act_api.c:377) [ 1675.466905] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1675.467121] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1675.467328] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1675.467618] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1675.467879] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1675.468126] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1675.468339] tcf_chain_flush (net/sched/cls_api.c:737) [ 1675.468562] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1675.468777] tcf_block_put (net/sched/cls_api.c:1543) [ 1675.468973] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1675.469216] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1675.469433] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1675.469634] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1675.469924] default_device_exit_batch (net/core/dev.c:11592) [ 1675.470181] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1675.470383] ? process_one_work (kernel/workqueue.c:2606) [ 1675.470613] process_one_work (kernel/workqueue.c:2633) [ 1675.470837] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1675.471040] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1675.471269] kthread (kernel/kthread.c:388) [ 1675.471449] ? __pfx_kthread (kernel/kthread.c:341) [ 1675.471659] ret_from_fork (arch/x86/kernel/process.c:147) [ 1675.471858] ? __pfx_kthread (kernel/kthread.c:341) [ 1675.472060] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1717.312622] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 1706313 jiffies s: 4157 root: 0x8/. | [ 1717.313243] rcu: blocking rcu_node structures (internal RCU debug): | [ 1717.313574] Sending NMI from CPU 2 to CPUs 3: | [ 1717.313829] NMI backtrace for cpu 3 [ 1717.313835] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1717.313837] Workqueue: netns cleanup_net [ 1717.313844] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1717.313851] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1717.313852] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1717.313854] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1717.313855] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1717.313856] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1717.313857] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1717.313858] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1717.313860] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1717.313861] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1717.313862] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1717.313863] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1717.313863] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1717.313864] PKRU: 55555554 [ 1717.313865] Call Trace: [ 1717.313867] [ 1717.313870] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 1717.313875] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 1717.313879] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 1717.313882] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 1717.313887] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 1717.313889] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 1717.313894] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1717.313896] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1717.313899] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1717.313901] [ 1717.313901] [ 1717.313902] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1717.313908] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1717.313910] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1717.313914] ? up_read (kernel/locking/rwsem.c:1622) [ 1717.313918] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1717.313923] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1717.313924] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1717.313927] __tcf_action_put (net/sched/act_api.c:377) [ 1717.313932] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1717.313935] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1717.313937] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1717.313944] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1717.313950] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1717.313955] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1717.313958] tcf_chain_flush (net/sched/cls_api.c:737) [ 1717.313960] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1717.313962] tcf_block_put (net/sched/cls_api.c:1543) [ 1717.313965] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1717.313970] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1717.313975] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1717.313978] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1717.313982] default_device_exit_batch (net/core/dev.c:11592) [ 1717.313985] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1717.313988] ? process_one_work (kernel/workqueue.c:2606) [ 1717.313992] process_one_work (kernel/workqueue.c:2633) [ 1717.313995] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1717.313998] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1717.313999] kthread (kernel/kthread.c:388) [ 1717.314002] ? __pfx_kthread (kernel/kthread.c:341) [ 1717.314004] ret_from_fork (arch/x86/kernel/process.c:147) [ 1717.314008] ? __pfx_kthread (kernel/kthread.c:341) [ 1717.314009] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1717.314014] | [ 1753.475609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1753.475947] rcu: 3-....: (1741835 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=400538 | [ 1753.476467] rcu: (t=1742506 jiffies g=7477 q=1773 ncpus=4) [ 1753.477214] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1753.477810] Workqueue: netns cleanup_net [ 1753.478028] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1753.478329] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1753.479294] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1753.479571] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1753.479947] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1753.480321] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1753.480695] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1753.481070] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1753.481447] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1753.481872] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1753.482176] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1753.482552] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1753.482927] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1753.483300] PKRU: 55555554 [ 1753.483449] Call Trace: [ 1753.483586] [ 1753.483704] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1753.483942] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1753.484188] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 1753.484401] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 1753.484612] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 1753.484820] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1753.485058] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1753.485294] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1753.485579] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1753.485801] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1753.486057] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1753.486295] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1753.486528] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1753.486807] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1753.487072] [ 1753.487190] [ 1753.487309] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1753.487593] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1753.487854] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1753.488063] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1753.488268] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1753.488536] ? up_read (kernel/locking/rwsem.c:1622) [ 1753.488713] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1753.488960] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1753.489228] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1753.489484] __tcf_action_put (net/sched/act_api.c:377) [ 1753.489695] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1753.489913] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1753.490121] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1753.490402] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1753.490659] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1753.490907] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1753.491119] tcf_chain_flush (net/sched/cls_api.c:737) [ 1753.491326] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1753.491533] tcf_block_put (net/sched/cls_api.c:1543) [ 1753.491728] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1753.491971] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1753.492180] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1753.492375] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1753.492664] default_device_exit_batch (net/core/dev.c:11592) [ 1753.492922] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1753.493117] ? process_one_work (kernel/workqueue.c:2606) [ 1753.493345] process_one_work (kernel/workqueue.c:2633) [ 1753.493562] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1753.493765] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1753.493997] kthread (kernel/kthread.c:388) [ 1753.494172] ? __pfx_kthread (kernel/kthread.c:341) [ 1753.494375] ret_from_fork (arch/x86/kernel/process.c:147) [ 1753.494572] ? __pfx_kthread (kernel/kthread.c:341) [ 1753.494779] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1815.616638] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 1804617 jiffies s: 4157 root: 0x8/. | [ 1815.617526] rcu: blocking rcu_node structures (internal RCU debug): | [ 1815.618023] Sending NMI from CPU 2 to CPUs 3: | [ 1815.618379] NMI backtrace for cpu 3 [ 1815.618385] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1815.618387] Workqueue: netns cleanup_net [ 1815.618396] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1815.618402] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1815.618404] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1815.618405] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1815.618407] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1815.618408] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1815.618409] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1815.618410] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1815.618412] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1815.618413] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1815.618414] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1815.618415] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1815.618416] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1815.618416] PKRU: 55555554 [ 1815.618417] Call Trace: [ 1815.618420] [ 1815.618422] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 1815.618427] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 1815.618432] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 1815.618435] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 1815.618439] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 1815.618441] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 1815.618447] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1815.618450] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1815.618452] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1815.618454] [ 1815.618455] [ 1815.618456] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1815.618461] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1815.618463] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1815.618468] ? up_read (kernel/locking/rwsem.c:1622) [ 1815.618471] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1815.618477] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1815.618478] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1815.618480] __tcf_action_put (net/sched/act_api.c:377) [ 1815.618486] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1815.618488] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1815.618491] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1815.618497] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1815.618503] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1815.618509] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1815.618512] tcf_chain_flush (net/sched/cls_api.c:737) [ 1815.618513] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1815.618516] tcf_block_put (net/sched/cls_api.c:1543) [ 1815.618518] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1815.618523] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1815.618528] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1815.618531] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1815.618535] default_device_exit_batch (net/core/dev.c:11592) [ 1815.618538] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1815.618541] ? process_one_work (kernel/workqueue.c:2606) [ 1815.618546] process_one_work (kernel/workqueue.c:2633) [ 1815.618549] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1815.618551] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1815.618553] kthread (kernel/kthread.c:388) [ 1815.618556] ? __pfx_kthread (kernel/kthread.c:341) [ 1815.618558] ret_from_fork (arch/x86/kernel/process.c:147) [ 1815.618562] ? __pfx_kthread (kernel/kthread.c:341) [ 1815.618563] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1815.618568] | [ 1831.498609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1831.498940] rcu: 3-....: (1819825 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=418576 | [ 1831.499459] rcu: (t=1820529 jiffies g=7477 q=1773 ncpus=4) [ 1831.500203] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1831.500795] Workqueue: netns cleanup_net [ 1831.501012] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1831.501303] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1831.502275] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1831.502551] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1831.502926] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1831.503298] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1831.503671] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1831.504044] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1831.504420] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1831.504843] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1831.505147] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1831.505522] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1831.505894] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1831.506267] PKRU: 55555554 [ 1831.506416] Call Trace: [ 1831.506552] [ 1831.506667] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1831.506906] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1831.507150] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 1831.507361] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 1831.507569] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 1831.507782] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1831.508020] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1831.508255] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1831.508536] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1831.508756] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1831.509010] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1831.509250] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1831.509481] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1831.509762] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1831.510026] [ 1831.510145] [ 1831.510263] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1831.510544] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1831.510808] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1831.511014] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1831.511212] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1831.511477] ? up_read (kernel/locking/rwsem.c:1622) [ 1831.511655] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1831.511896] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1831.512159] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1831.512416] __tcf_action_put (net/sched/act_api.c:377) [ 1831.512625] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1831.512840] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1831.513047] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1831.513329] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1831.513584] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1831.513828] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1831.514042] tcf_chain_flush (net/sched/cls_api.c:737) [ 1831.514247] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1831.514453] tcf_block_put (net/sched/cls_api.c:1543) [ 1831.514648] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1831.514889] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1831.515097] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1831.515291] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1831.515578] default_device_exit_batch (net/core/dev.c:11592) [ 1831.515835] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1831.516029] ? process_one_work (kernel/workqueue.c:2606) [ 1831.516256] process_one_work (kernel/workqueue.c:2633) [ 1831.516471] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1831.516674] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1831.516905] kthread (kernel/kthread.c:388) [ 1831.517076] ? __pfx_kthread (kernel/kthread.c:341) [ 1831.517278] ret_from_fork (arch/x86/kernel/process.c:147) [ 1831.517473] ? __pfx_kthread (kernel/kthread.c:341) [ 1831.517681] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1831.517896] | [ 1909.521611] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1909.521963] rcu: 3-....: (1897830 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=436519 | [ 1909.522491] rcu: (t=1898552 jiffies g=7477 q=1773 ncpus=4) [ 1909.523249] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1909.523861] Workqueue: netns cleanup_net [ 1909.524083] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 1909.524385] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 <8b> 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: 90 nop f: 90 nop 10: 90 nop 11: f3 0f 1e fa endbr64 15: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 1a: 41 55 push %r13 1c: 41 54 push %r12 1e: 55 push %rbp 1f: 48 89 fd mov %rdi,%rbp 22: 53 push %rbx 23: 66 90 xchg %ax,%ax 25: ba 01 00 00 00 mov $0x1,%edx 2a:* 8b 45 00 mov 0x0(%rbp),%eax <-- trapping instruction 2d: 85 c0 test %eax,%eax 2f: 75 14 jne 0x45 31: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 36: 85 c0 test %eax,%eax 38: 75 f0 jne 0x2a 3a: 5b pop %rbx 3b: 5d pop %rbp 3c: 41 5c pop %r12 3e: 41 5d pop %r13 Code starting with the faulting instruction =========================================== 0: 8b 45 00 mov 0x0(%rbp),%eax 3: 85 c0 test %eax,%eax 5: 75 14 jne 0x1b 7: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) c: 85 c0 test %eax,%eax e: 75 f0 jne 0x0 10: 5b pop %rbx 11: 5d pop %rbp 12: 41 5c pop %r12 14: 41 5d pop %r13 [ 1909.525372] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1909.525660] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1909.526041] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1909.526419] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1909.526807] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1909.527182] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1909.527587] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1909.528020] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1909.528327] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1909.528718] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1909.529090] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1909.529473] PKRU: 55555554 [ 1909.529627] Call Trace: [ 1909.529771] [ 1909.529887] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1909.530127] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1909.530376] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 1909.530589] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 1909.530811] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 1909.531018] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1909.531260] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1909.531505] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1909.531799] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1909.532019] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1909.532273] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1909.532520] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1909.532762] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1909.533040] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1909.533304] [ 1909.533429] [ 1909.533548] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1909.533843] ? queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 1909.534105] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1909.534313] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1909.534518] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1909.534797] ? up_read (kernel/locking/rwsem.c:1622) [ 1909.534977] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1909.535219] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1909.535490] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1909.535759] __tcf_action_put (net/sched/act_api.c:377) [ 1909.535969] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1909.536183] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1909.536396] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1909.536692] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1909.536947] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1909.537192] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1909.537420] tcf_chain_flush (net/sched/cls_api.c:737) [ 1909.537634] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1909.537846] tcf_block_put (net/sched/cls_api.c:1543) [ 1909.538042] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1909.538288] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1909.538503] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1909.538713] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1909.539000] default_device_exit_batch (net/core/dev.c:11592) [ 1909.539257] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1909.539460] ? process_one_work (kernel/workqueue.c:2606) [ 1909.539699] process_one_work (kernel/workqueue.c:2633) [ 1909.539919] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1909.540122] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1909.540352] kthread (kernel/kthread.c:388) [ 1909.540532] ? __pfx_kthread (kernel/kthread.c:341) [ 1909.540744] ret_from_fork (arch/x86/kernel/process.c:147) [ 1909.540941] ? __pfx_kthread (kernel/kthread.c:341) [ 1909.541143] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1913.920635] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 1902921 jiffies s: 4157 root: 0x8/. | [ 1913.921514] rcu: blocking rcu_node structures (internal RCU debug): | [ 1913.922023] Sending NMI from CPU 2 to CPUs 3: | [ 1913.922362] NMI backtrace for cpu 3 [ 1913.922370] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1913.922372] Workqueue: netns cleanup_net [ 1913.922381] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1913.922389] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1913.922390] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1913.922392] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1913.922393] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1913.922394] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1913.922395] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1913.922396] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1913.922399] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1913.922400] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1913.922401] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1913.922402] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1913.922402] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1913.922403] PKRU: 55555554 [ 1913.922404] Call Trace: [ 1913.922407] [ 1913.922409] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 1913.922414] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 1913.922419] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 1913.922422] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 1913.922427] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 1913.922429] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 1913.922434] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1913.922436] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1913.922439] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1913.922441] [ 1913.922441] [ 1913.922442] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1913.922448] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1913.922449] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1913.922455] ? up_read (kernel/locking/rwsem.c:1622) [ 1913.922458] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1913.922463] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1913.922465] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1913.922467] __tcf_action_put (net/sched/act_api.c:377) [ 1913.922473] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1913.922475] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1913.922478] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1913.922484] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1913.922490] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1913.922495] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1913.922498] tcf_chain_flush (net/sched/cls_api.c:737) [ 1913.922499] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1913.922501] tcf_block_put (net/sched/cls_api.c:1543) [ 1913.922504] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1913.922509] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1913.922514] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1913.922517] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1913.922521] default_device_exit_batch (net/core/dev.c:11592) [ 1913.922524] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1913.922527] ? process_one_work (kernel/workqueue.c:2606) [ 1913.922531] process_one_work (kernel/workqueue.c:2633) [ 1913.922533] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1913.922536] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1913.922537] kthread (kernel/kthread.c:388) [ 1913.922540] ? __pfx_kthread (kernel/kthread.c:341) [ 1913.922541] ret_from_fork (arch/x86/kernel/process.c:147) [ 1913.922545] ? __pfx_kthread (kernel/kthread.c:341) [ 1913.922546] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1913.922552] | [ 1987.544609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1987.544951] rcu: 3-....: (1975820 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=454634 | [ 1987.545470] rcu: (t=1976575 jiffies g=7477 q=1774 ncpus=4) [ 1987.546214] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1987.546806] Workqueue: netns cleanup_net [ 1987.547091] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1987.547386] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1987.548346] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 1987.548623] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 1987.548999] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 1987.549370] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 1987.549744] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 1987.550117] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 1987.550491] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 1987.550923] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1987.551228] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 1987.551603] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1987.551976] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1987.552348] PKRU: 55555554 [ 1987.552497] Call Trace: [ 1987.552634] [ 1987.552749] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1987.552988] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1987.553232] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 1987.553444] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 1987.553652] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 1987.553863] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1987.554104] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1987.554340] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1987.554623] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1987.554843] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1987.555097] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1987.555336] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1987.555568] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1987.555848] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1987.556112] [ 1987.556231] [ 1987.556350] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1987.556632] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1987.556898] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1987.557105] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1987.557304] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1987.557571] ? up_read (kernel/locking/rwsem.c:1622) [ 1987.557751] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1987.557993] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1987.558257] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1987.558513] __tcf_action_put (net/sched/act_api.c:377) [ 1987.558723] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1987.558938] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1987.559146] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1987.559430] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1987.559687] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1987.559932] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1987.560146] tcf_chain_flush (net/sched/cls_api.c:737) [ 1987.560353] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1987.560559] tcf_block_put (net/sched/cls_api.c:1543) [ 1987.560755] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1987.561001] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1987.561210] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1987.561406] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 1987.561693] default_device_exit_batch (net/core/dev.c:11592) [ 1987.561951] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1987.562146] ? process_one_work (kernel/workqueue.c:2606) [ 1987.562371] process_one_work (kernel/workqueue.c:2633) [ 1987.562587] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1987.562791] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1987.563022] kthread (kernel/kthread.c:388) [ 1987.563193] ? __pfx_kthread (kernel/kthread.c:341) [ 1987.563396] ret_from_fork (arch/x86/kernel/process.c:147) [ 1987.563592] ? __pfx_kthread (kernel/kthread.c:341) [ 1987.563794] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2012.224622] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2001225 jiffies s: 4157 root: 0x8/. | [ 2012.225239] rcu: blocking rcu_node structures (internal RCU debug): | [ 2012.225570] Sending NMI from CPU 2 to CPUs 3: | [ 2012.225826] NMI backtrace for cpu 3 [ 2012.225833] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2012.225835] Workqueue: netns cleanup_net [ 2012.225843] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2012.225850] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2012.225851] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2012.225853] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2012.225854] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2012.225855] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2012.225856] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2012.225857] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2012.225860] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2012.225862] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2012.225862] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2012.225863] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2012.225864] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2012.225864] PKRU: 55555554 [ 2012.225865] Call Trace: [ 2012.225867] [ 2012.225870] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 2012.225875] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 2012.225880] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 2012.225883] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 2012.225888] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 2012.225890] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 2012.225895] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2012.225897] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2012.225899] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2012.225901] [ 2012.225902] [ 2012.225903] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2012.225908] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2012.225910] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2012.225915] ? up_read (kernel/locking/rwsem.c:1622) [ 2012.225919] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2012.225924] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2012.225926] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2012.225929] __tcf_action_put (net/sched/act_api.c:377) [ 2012.225934] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2012.225936] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2012.225939] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2012.225946] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2012.225952] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2012.225957] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2012.225960] tcf_chain_flush (net/sched/cls_api.c:737) [ 2012.225962] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2012.225964] tcf_block_put (net/sched/cls_api.c:1543) [ 2012.225967] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2012.225972] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2012.225976] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2012.225979] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2012.225984] default_device_exit_batch (net/core/dev.c:11592) [ 2012.225987] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2012.225990] ? process_one_work (kernel/workqueue.c:2606) [ 2012.225994] process_one_work (kernel/workqueue.c:2633) [ 2012.225997] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2012.226000] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2012.226002] kthread (kernel/kthread.c:388) [ 2012.226005] ? __pfx_kthread (kernel/kthread.c:341) [ 2012.226006] ret_from_fork (arch/x86/kernel/process.c:147) [ 2012.226010] ? __pfx_kthread (kernel/kthread.c:341) [ 2012.226011] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2012.226016] | [ 2065.567609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2065.567958] rcu: 3-....: (2053810 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=472216 | [ 2065.568478] rcu: (t=2054598 jiffies g=7477 q=1774 ncpus=4) [ 2065.569225] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2065.569820] Workqueue: netns cleanup_net [ 2065.570040] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2065.570335] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2065.571302] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2065.571579] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2065.571958] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2065.572334] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2065.572711] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2065.573088] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2065.573466] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2065.573889] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2065.574195] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2065.574571] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2065.574947] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2065.575322] PKRU: 55555554 [ 2065.575470] Call Trace: [ 2065.575609] [ 2065.575725] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2065.575966] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2065.576211] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 2065.576424] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 2065.576697] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 2065.576905] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2065.577145] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2065.577393] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2065.577676] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2065.577899] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2065.578155] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2065.578396] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2065.578627] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2065.578908] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2065.579173] [ 2065.579292] [ 2065.579412] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2065.579697] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2065.579958] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2065.580167] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2065.580366] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2065.580634] ? up_read (kernel/locking/rwsem.c:1622) [ 2065.580815] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2065.581058] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2065.581322] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2065.581578] __tcf_action_put (net/sched/act_api.c:377) [ 2065.581788] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2065.582003] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2065.582211] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2065.582493] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2065.582750] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2065.582996] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2065.583210] tcf_chain_flush (net/sched/cls_api.c:737) [ 2065.583418] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2065.583625] tcf_block_put (net/sched/cls_api.c:1543) [ 2065.583821] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2065.584064] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2065.584274] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2065.584473] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2065.584761] default_device_exit_batch (net/core/dev.c:11592) [ 2065.585021] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2065.585216] ? process_one_work (kernel/workqueue.c:2606) [ 2065.585445] process_one_work (kernel/workqueue.c:2633) [ 2065.585662] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2065.585865] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2065.586096] kthread (kernel/kthread.c:388) [ 2065.586269] ? __pfx_kthread (kernel/kthread.c:341) [ 2065.586472] ret_from_fork (arch/x86/kernel/process.c:147) [ 2065.586675] ? __pfx_kthread (kernel/kthread.c:341) [ 2065.586879] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2110.528620] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2099529 jiffies s: 4157 root: 0x8/. | [ 2110.529235] rcu: blocking rcu_node structures (internal RCU debug): | [ 2110.529566] Sending NMI from CPU 2 to CPUs 3: | [ 2110.529816] NMI backtrace for cpu 3 [ 2110.529822] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2110.529825] Workqueue: netns cleanup_net [ 2110.529832] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2110.529839] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2110.529841] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2110.529842] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2110.529844] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2110.529845] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2110.529846] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2110.529847] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2110.529849] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2110.529850] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2110.529851] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2110.529852] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2110.529853] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2110.529854] PKRU: 55555554 [ 2110.529855] Call Trace: [ 2110.529858] [ 2110.529860] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 2110.529865] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 2110.529870] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 2110.529873] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 2110.529878] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 2110.529880] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 2110.529885] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2110.529887] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2110.529890] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2110.529892] [ 2110.529892] [ 2110.529893] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2110.529899] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2110.529901] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2110.529907] ? up_read (kernel/locking/rwsem.c:1622) [ 2110.529910] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2110.529915] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2110.529917] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2110.529919] __tcf_action_put (net/sched/act_api.c:377) [ 2110.529924] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2110.529927] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2110.529929] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2110.529936] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2110.529942] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2110.529947] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2110.529951] tcf_chain_flush (net/sched/cls_api.c:737) [ 2110.529953] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2110.529955] tcf_block_put (net/sched/cls_api.c:1543) [ 2110.529958] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2110.529963] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2110.529968] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2110.529971] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2110.529975] default_device_exit_batch (net/core/dev.c:11592) [ 2110.529978] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2110.529981] ? process_one_work (kernel/workqueue.c:2606) [ 2110.529985] process_one_work (kernel/workqueue.c:2633) [ 2110.529989] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2110.529991] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2110.529993] kthread (kernel/kthread.c:388) [ 2110.529995] ? __pfx_kthread (kernel/kthread.c:341) [ 2110.529997] ret_from_fork (arch/x86/kernel/process.c:147) [ 2110.530001] ? __pfx_kthread (kernel/kthread.c:341) [ 2110.530002] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2110.530008] | [ 2143.589610] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2143.589949] rcu: 3-....: (2131799 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=489657 | [ 2143.590470] rcu: (t=2132620 jiffies g=7477 q=1775 ncpus=4) [ 2143.591216] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2143.591812] Workqueue: netns cleanup_net [ 2143.592030] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2143.592324] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2143.593289] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2143.593566] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2143.593941] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2143.594315] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2143.594687] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2143.595061] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2143.595437] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2143.595859] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2143.596169] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2143.596545] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2143.596920] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2143.597292] PKRU: 55555554 [ 2143.597441] Call Trace: [ 2143.597580] [ 2143.597697] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2143.597935] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2143.598179] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 2143.598391] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 2143.598600] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 2143.598808] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2143.599045] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2143.599282] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2143.599565] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2143.599786] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2143.600043] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2143.600281] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2143.600514] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2143.600792] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2143.601056] [ 2143.601177] [ 2143.601297] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2143.601582] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2143.601843] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2143.602053] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2143.602251] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2143.602518] ? up_read (kernel/locking/rwsem.c:1622) [ 2143.602696] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2143.602938] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2143.603203] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2143.603459] __tcf_action_put (net/sched/act_api.c:377) [ 2143.603671] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2143.603887] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2143.604096] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2143.604377] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2143.604635] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2143.604882] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2143.605094] tcf_chain_flush (net/sched/cls_api.c:737) [ 2143.605300] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2143.605507] tcf_block_put (net/sched/cls_api.c:1543) [ 2143.605701] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2143.605944] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2143.606160] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2143.606356] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2143.606644] default_device_exit_batch (net/core/dev.c:11592) [ 2143.606900] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2143.607095] ? process_one_work (kernel/workqueue.c:2606) [ 2143.607324] process_one_work (kernel/workqueue.c:2633) [ 2143.607540] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2143.607743] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2143.607975] kthread (kernel/kthread.c:388) [ 2143.608150] ? __pfx_kthread (kernel/kthread.c:341) [ 2143.608354] ret_from_fork (arch/x86/kernel/process.c:147) [ 2143.608550] ? __pfx_kthread (kernel/kthread.c:341) [ 2143.608753] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2208.832624] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2197833 jiffies s: 4157 root: 0x8/. | [ 2208.833255] rcu: blocking rcu_node structures (internal RCU debug): | [ 2208.833587] Sending NMI from CPU 2 to CPUs 3: | [ 2208.833844] NMI backtrace for cpu 3 [ 2208.833850] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2208.833852] Workqueue: netns cleanup_net [ 2208.833861] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2208.833867] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2208.833869] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2208.833870] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2208.833871] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2208.833872] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2208.833873] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2208.833874] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2208.833877] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2208.833878] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2208.833879] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2208.833880] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2208.833880] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2208.833881] PKRU: 55555554 [ 2208.833881] Call Trace: [ 2208.833884] [ 2208.833887] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 2208.833892] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 2208.833897] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 2208.833900] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 2208.833904] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 2208.833906] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 2208.833911] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2208.833914] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2208.833916] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2208.833918] [ 2208.833919] [ 2208.833920] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2208.833926] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2208.833928] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2208.833933] ? up_read (kernel/locking/rwsem.c:1622) [ 2208.833936] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2208.833941] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2208.833942] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2208.833945] __tcf_action_put (net/sched/act_api.c:377) [ 2208.833956] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2208.833958] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2208.833961] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2208.833968] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2208.833973] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2208.833979] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2208.833982] tcf_chain_flush (net/sched/cls_api.c:737) [ 2208.833984] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2208.833986] tcf_block_put (net/sched/cls_api.c:1543) [ 2208.833988] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2208.833993] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2208.833998] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2208.834001] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2208.834006] default_device_exit_batch (net/core/dev.c:11592) [ 2208.834009] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2208.834012] ? process_one_work (kernel/workqueue.c:2606) [ 2208.834015] process_one_work (kernel/workqueue.c:2633) [ 2208.834018] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2208.834020] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2208.834022] kthread (kernel/kthread.c:388) [ 2208.834025] ? __pfx_kthread (kernel/kthread.c:341) [ 2208.834026] ret_from_fork (arch/x86/kernel/process.c:147) [ 2208.834030] ? __pfx_kthread (kernel/kthread.c:341) [ 2208.834031] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2208.834038] | [ 2221.612609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2221.612951] rcu: 3-....: (2209789 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=506652 | [ 2221.613472] rcu: (t=2210643 jiffies g=7477 q=1775 ncpus=4) [ 2221.614219] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2221.614816] Workqueue: netns cleanup_net [ 2221.615037] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 2221.615332] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 <85> c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: f3 0f 1e fa endbr64 12: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 17: 41 55 push %r13 19: 41 54 push %r12 1b: 55 push %rbp 1c: 48 89 fd mov %rdi,%rbp 1f: 53 push %rbx 20: 66 90 xchg %ax,%ax 22: ba 01 00 00 00 mov $0x1,%edx 27: 8b 45 00 mov 0x0(%rbp),%eax 2a:* 85 c0 test %eax,%eax <-- trapping instruction 2c: 75 14 jne 0x42 2e: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 33: 85 c0 test %eax,%eax 35: 75 f0 jne 0x27 37: 5b pop %rbx 38: 5d pop %rbp 39: 41 5c pop %r12 3b: 41 5d pop %r13 3d: c3 ret 3e: cc int3 3f: cc int3 Code starting with the faulting instruction =========================================== 0: 85 c0 test %eax,%eax 2: 75 14 jne 0x18 4: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 9: 85 c0 test %eax,%eax b: 75 f0 jne 0xfffffffffffffffd d: 5b pop %rbx e: 5d pop %rbp f: 41 5c pop %r12 11: 41 5d pop %r13 13: c3 ret 14: cc int3 15: cc int3 [ 2221.616300] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2221.616576] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2221.616952] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2221.617324] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2221.617703] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2221.618077] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2221.618454] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2221.618880] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2221.619185] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2221.619561] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2221.619935] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2221.620307] PKRU: 55555554 [ 2221.620456] Call Trace: [ 2221.620594] [ 2221.620710] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2221.620951] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2221.621197] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 2221.621409] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 2221.621617] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 2221.621827] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2221.622066] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2221.622303] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2221.622586] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2221.622807] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2221.623062] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2221.623304] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2221.623535] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2221.623814] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2221.624079] [ 2221.624198] [ 2221.624318] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2221.624603] ? queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 2221.624866] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2221.625073] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2221.625271] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2221.625537] ? up_read (kernel/locking/rwsem.c:1622) [ 2221.625724] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2221.625968] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2221.626234] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2221.626491] __tcf_action_put (net/sched/act_api.c:377) [ 2221.626701] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2221.626916] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2221.627125] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2221.627408] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2221.627664] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2221.627910] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2221.628123] tcf_chain_flush (net/sched/cls_api.c:737) [ 2221.628330] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2221.628538] tcf_block_put (net/sched/cls_api.c:1543) [ 2221.628734] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2221.628977] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2221.629187] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2221.629383] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2221.629670] default_device_exit_batch (net/core/dev.c:11592) [ 2221.629928] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2221.630124] ? process_one_work (kernel/workqueue.c:2606) [ 2221.630351] process_one_work (kernel/workqueue.c:2633) [ 2221.630568] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2221.630772] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2221.631004] kthread (kernel/kthread.c:388) [ 2221.631176] ? __pfx_kthread (kernel/kthread.c:341) [ 2221.631378] ret_from_fork (arch/x86/kernel/process.c:147) [ 2221.631575] ? __pfx_kthread (kernel/kthread.c:341) [ 2221.631783] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2221.632000] | [ 2299.635609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2299.635952] rcu: 3-....: (2287794 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=523963 | [ 2299.636468] rcu: (t=2288666 jiffies g=7477 q=1775 ncpus=4) [ 2299.637212] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2299.637805] Workqueue: netns cleanup_net [ 2299.638023] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2299.638318] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2299.639280] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2299.639558] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2299.639932] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2299.640305] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2299.640681] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2299.641054] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2299.641429] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2299.641858] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2299.642163] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2299.642539] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2299.642914] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2299.643290] PKRU: 55555554 [ 2299.643437] Call Trace: [ 2299.643575] [ 2299.643692] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2299.643933] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2299.644182] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 2299.644395] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 2299.644604] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 2299.644811] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2299.645051] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2299.645294] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2299.645600] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2299.645819] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2299.646075] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2299.646315] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2299.646546] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2299.646826] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2299.647090] [ 2299.647208] [ 2299.647326] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2299.647610] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2299.647873] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2299.648080] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2299.648278] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2299.648544] ? up_read (kernel/locking/rwsem.c:1622) [ 2299.648724] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2299.648966] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2299.649230] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2299.649487] __tcf_action_put (net/sched/act_api.c:377) [ 2299.649697] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2299.649913] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2299.650119] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2299.650404] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2299.650663] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2299.650926] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2299.651141] tcf_chain_flush (net/sched/cls_api.c:737) [ 2299.651349] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2299.651556] tcf_block_put (net/sched/cls_api.c:1543) [ 2299.651752] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2299.651993] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2299.652204] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2299.652399] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2299.652687] default_device_exit_batch (net/core/dev.c:11592) [ 2299.652943] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2299.653139] ? process_one_work (kernel/workqueue.c:2606) [ 2299.653364] process_one_work (kernel/workqueue.c:2633) [ 2299.653583] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2299.653787] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2299.654017] kthread (kernel/kthread.c:388) [ 2299.654190] ? __pfx_kthread (kernel/kthread.c:341) [ 2299.654392] ret_from_fork (arch/x86/kernel/process.c:147) [ 2299.654588] ? __pfx_kthread (kernel/kthread.c:341) [ 2299.654790] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2307.136622] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2296137 jiffies s: 4157 root: 0x8/. | [ 2307.137244] rcu: blocking rcu_node structures (internal RCU debug): | [ 2307.137577] Sending NMI from CPU 2 to CPUs 3: | [ 2307.137833] NMI backtrace for cpu 3 [ 2307.137841] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2307.137843] Workqueue: netns cleanup_net [ 2307.137851] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2307.137858] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2307.137859] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2307.137861] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2307.137862] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2307.137863] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2307.137864] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2307.137865] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2307.137868] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2307.137870] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2307.137870] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2307.137871] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2307.137872] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2307.137873] PKRU: 55555554 [ 2307.137874] Call Trace: [ 2307.137877] [ 2307.137879] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 2307.137884] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 2307.137889] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 2307.137892] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 2307.137896] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 2307.137898] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 2307.137904] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2307.137906] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2307.137909] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2307.137911] [ 2307.137911] [ 2307.137912] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2307.137917] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2307.137919] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2307.137924] ? up_read (kernel/locking/rwsem.c:1622) [ 2307.137927] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2307.137932] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2307.137933] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2307.137935] __tcf_action_put (net/sched/act_api.c:377) [ 2307.137940] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2307.137943] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2307.137945] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2307.137951] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2307.137958] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2307.137963] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2307.137966] tcf_chain_flush (net/sched/cls_api.c:737) [ 2307.137967] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2307.137970] tcf_block_put (net/sched/cls_api.c:1543) [ 2307.137972] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2307.137976] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2307.137980] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2307.137983] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2307.137987] default_device_exit_batch (net/core/dev.c:11592) [ 2307.137990] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2307.137992] ? process_one_work (kernel/workqueue.c:2606) [ 2307.137996] process_one_work (kernel/workqueue.c:2633) [ 2307.137999] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2307.138002] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2307.138003] kthread (kernel/kthread.c:388) [ 2307.138005] ? __pfx_kthread (kernel/kthread.c:341) [ 2307.138007] ret_from_fork (arch/x86/kernel/process.c:147) [ 2307.138011] ? __pfx_kthread (kernel/kthread.c:341) [ 2307.138012] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2307.138017] | [ 2377.658609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2377.658949] rcu: 3-....: (2365784 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=541588 | [ 2377.659466] rcu: (t=2366689 jiffies g=7477 q=1775 ncpus=4) [ 2377.660216] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2377.660813] Workqueue: netns cleanup_net [ 2377.661030] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2377.661325] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2377.662285] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2377.662560] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2377.662936] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2377.663310] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2377.663684] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2377.664056] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2377.664432] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2377.664856] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2377.665166] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2377.665542] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2377.665918] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2377.666292] PKRU: 55555554 [ 2377.666443] Call Trace: [ 2377.666580] [ 2377.666697] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2377.666937] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2377.667181] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 2377.667395] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 2377.667604] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 2377.667814] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2377.668054] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2377.668291] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2377.668589] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2377.668811] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2377.669068] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2377.669309] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2377.669540] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2377.669818] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2377.670083] [ 2377.670202] [ 2377.670320] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2377.670604] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2377.670866] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2377.671076] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2377.671274] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2377.671543] ? up_read (kernel/locking/rwsem.c:1622) [ 2377.671722] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2377.671964] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2377.672227] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2377.672484] __tcf_action_put (net/sched/act_api.c:377) [ 2377.672693] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2377.672911] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2377.673118] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2377.673399] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2377.673655] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2377.673902] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2377.674113] tcf_chain_flush (net/sched/cls_api.c:737) [ 2377.674319] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2377.674526] tcf_block_put (net/sched/cls_api.c:1543) [ 2377.674721] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2377.674969] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2377.675177] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2377.675372] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2377.675661] default_device_exit_batch (net/core/dev.c:11592) [ 2377.675918] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2377.676115] ? process_one_work (kernel/workqueue.c:2606) [ 2377.676340] process_one_work (kernel/workqueue.c:2633) [ 2377.676555] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2377.676760] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2377.676990] kthread (kernel/kthread.c:388) [ 2377.677166] ? __pfx_kthread (kernel/kthread.c:341) [ 2377.677368] ret_from_fork (arch/x86/kernel/process.c:147) [ 2377.677563] ? __pfx_kthread (kernel/kthread.c:341) [ 2377.677765] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2405.440619] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2394441 jiffies s: 4157 root: 0x8/. | [ 2405.441234] rcu: blocking rcu_node structures (internal RCU debug): | [ 2405.441566] Sending NMI from CPU 2 to CPUs 3: | [ 2405.441819] NMI backtrace for cpu 3 [ 2405.441826] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2405.441827] Workqueue: netns cleanup_net [ 2405.441834] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2405.441838] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2405.441839] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2405.441841] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2405.441842] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2405.441843] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2405.441844] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2405.441845] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2405.441848] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2405.441849] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2405.441850] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2405.441850] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2405.441851] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2405.441852] PKRU: 55555554 [ 2405.441852] Call Trace: [ 2405.441855] [ 2405.441857] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 2405.441862] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 2405.441866] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 2405.441869] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 2405.441873] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 2405.441875] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 2405.441879] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2405.441881] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2405.441883] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2405.441886] [ 2405.441886] [ 2405.441887] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2405.441892] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2405.441893] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2405.441897] ? up_read (kernel/locking/rwsem.c:1622) [ 2405.441899] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2405.441904] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2405.441906] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2405.441908] __tcf_action_put (net/sched/act_api.c:377) [ 2405.441921] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2405.441923] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2405.441925] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2405.441931] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2405.441936] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2405.441942] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2405.441944] tcf_chain_flush (net/sched/cls_api.c:737) [ 2405.441946] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2405.441948] tcf_block_put (net/sched/cls_api.c:1543) [ 2405.441950] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2405.441955] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2405.441958] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2405.441961] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2405.441964] default_device_exit_batch (net/core/dev.c:11592) [ 2405.441967] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2405.441970] ? process_one_work (kernel/workqueue.c:2606) [ 2405.441973] process_one_work (kernel/workqueue.c:2633) [ 2405.441976] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2405.441978] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2405.441980] kthread (kernel/kthread.c:388) [ 2405.441983] ? __pfx_kthread (kernel/kthread.c:341) [ 2405.441984] ret_from_fork (arch/x86/kernel/process.c:147) [ 2405.441988] ? __pfx_kthread (kernel/kthread.c:341) [ 2405.441989] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2405.441993] | [ 2455.681609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2455.681950] rcu: 3-....: (2443774 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=559383 | [ 2455.682468] rcu: (t=2444712 jiffies g=7477 q=1775 ncpus=4) [ 2455.683214] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2455.683810] Workqueue: netns cleanup_net [ 2455.684027] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2455.684325] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2455.685291] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2455.685568] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2455.685943] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2455.686318] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2455.686695] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2455.687070] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2455.687444] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2455.687868] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2455.688174] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2455.688548] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2455.688923] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2455.689297] PKRU: 55555554 [ 2455.689447] Call Trace: [ 2455.689583] [ 2455.689701] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2455.689942] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2455.690186] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 2455.690399] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 2455.690608] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 2455.690816] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2455.691055] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2455.691291] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2455.691577] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2455.691796] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2455.692053] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2455.692295] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2455.692526] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2455.692807] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2455.693073] [ 2455.693191] [ 2455.693309] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2455.693595] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2455.693858] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2455.694066] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2455.694265] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2455.694538] ? up_read (kernel/locking/rwsem.c:1622) [ 2455.694717] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2455.694959] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2455.695224] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2455.695480] __tcf_action_put (net/sched/act_api.c:377) [ 2455.695692] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2455.695907] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2455.696115] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2455.696399] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2455.696656] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2455.696904] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2455.697116] tcf_chain_flush (net/sched/cls_api.c:737) [ 2455.697324] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2455.697533] tcf_block_put (net/sched/cls_api.c:1543) [ 2455.697728] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2455.697969] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2455.698181] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2455.698378] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2455.698667] default_device_exit_batch (net/core/dev.c:11592) [ 2455.698925] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2455.699121] ? process_one_work (kernel/workqueue.c:2606) [ 2455.699348] process_one_work (kernel/workqueue.c:2633) [ 2455.699568] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2455.699773] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2455.700002] kthread (kernel/kthread.c:388) [ 2455.700178] ? __pfx_kthread (kernel/kthread.c:341) [ 2455.700381] ret_from_fork (arch/x86/kernel/process.c:147) [ 2455.700577] ? __pfx_kthread (kernel/kthread.c:341) [ 2455.700784] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2503.744638] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2492745 jiffies s: 4157 root: 0x8/. | [ 2503.745252] rcu: blocking rcu_node structures (internal RCU debug): | [ 2503.745593] Sending NMI from CPU 2 to CPUs 3: | [ 2503.745852] NMI backtrace for cpu 3 [ 2503.745861] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2503.745864] Workqueue: netns cleanup_net [ 2503.745874] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2503.745880] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2503.745881] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2503.745883] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2503.745884] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2503.745885] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2503.745886] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2503.745887] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2503.745890] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2503.745891] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2503.745892] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2503.745893] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2503.745893] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2503.745894] PKRU: 55555554 [ 2503.745895] Call Trace: [ 2503.745902] [ 2503.745908] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 2503.745913] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 2503.745918] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 2503.745921] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 2503.745925] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 2503.745928] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 2503.745933] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2503.745935] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2503.745937] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2503.745940] [ 2503.745940] [ 2503.745941] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2503.745947] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2503.745949] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2503.745954] ? up_read (kernel/locking/rwsem.c:1622) [ 2503.745958] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2503.745962] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2503.745964] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2503.745966] __tcf_action_put (net/sched/act_api.c:377) [ 2503.745971] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2503.745974] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2503.745976] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2503.745982] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2503.745988] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2503.745994] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2503.745997] tcf_chain_flush (net/sched/cls_api.c:737) [ 2503.745998] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2503.746000] tcf_block_put (net/sched/cls_api.c:1543) [ 2503.746003] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2503.746007] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2503.746012] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2503.746014] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2503.746018] default_device_exit_batch (net/core/dev.c:11592) [ 2503.746021] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2503.746024] ? process_one_work (kernel/workqueue.c:2606) [ 2503.746027] process_one_work (kernel/workqueue.c:2633) [ 2503.746030] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2503.746032] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2503.746034] kthread (kernel/kthread.c:388) [ 2503.746036] ? __pfx_kthread (kernel/kthread.c:341) [ 2503.746038] ret_from_fork (arch/x86/kernel/process.c:147) [ 2503.746041] ? __pfx_kthread (kernel/kthread.c:341) [ 2503.746043] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2503.746048] | [ 2533.704610] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2533.704968] rcu: 3-....: (2521764 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=577673 | [ 2533.705488] rcu: (t=2522735 jiffies g=7477 q=1775 ncpus=4) [ 2533.706239] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2533.706833] Workqueue: netns cleanup_net [ 2533.707055] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 2533.707351] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 <85> c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: f3 0f 1e fa endbr64 12: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 17: 41 55 push %r13 19: 41 54 push %r12 1b: 55 push %rbp 1c: 48 89 fd mov %rdi,%rbp 1f: 53 push %rbx 20: 66 90 xchg %ax,%ax 22: ba 01 00 00 00 mov $0x1,%edx 27: 8b 45 00 mov 0x0(%rbp),%eax 2a:* 85 c0 test %eax,%eax <-- trapping instruction 2c: 75 14 jne 0x42 2e: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 33: 85 c0 test %eax,%eax 35: 75 f0 jne 0x27 37: 5b pop %rbx 38: 5d pop %rbp 39: 41 5c pop %r12 3b: 41 5d pop %r13 3d: c3 ret 3e: cc int3 3f: cc int3 Code starting with the faulting instruction =========================================== 0: 85 c0 test %eax,%eax 2: 75 14 jne 0x18 4: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 9: 85 c0 test %eax,%eax b: 75 f0 jne 0xfffffffffffffffd d: 5b pop %rbx e: 5d pop %rbp f: 41 5c pop %r12 11: 41 5d pop %r13 13: c3 ret 14: cc int3 15: cc int3 [ 2533.708315] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2533.708598] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2533.708971] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2533.709345] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2533.709720] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2533.710094] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2533.710469] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2533.710893] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2533.711198] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2533.711572] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2533.711946] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2533.712320] PKRU: 55555554 [ 2533.712466] Call Trace: [ 2533.712608] [ 2533.712723] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2533.712964] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2533.713208] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 2533.713420] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 2533.713630] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 2533.713836] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2533.714075] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2533.714318] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2533.714600] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2533.714818] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2533.715074] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2533.715314] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2533.715545] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2533.715824] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2533.716089] [ 2533.716206] [ 2533.716325] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2533.716607] ? queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 2533.716868] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2533.717075] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2533.717273] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2533.717539] ? up_read (kernel/locking/rwsem.c:1622) [ 2533.717719] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2533.717962] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2533.718225] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2533.718481] __tcf_action_put (net/sched/act_api.c:377) [ 2533.718690] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2533.718904] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2533.719111] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2533.719394] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2533.719650] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2533.719896] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2533.720108] tcf_chain_flush (net/sched/cls_api.c:737) [ 2533.720316] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2533.720523] tcf_block_put (net/sched/cls_api.c:1543) [ 2533.720721] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2533.720963] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2533.721173] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2533.721369] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2533.721658] default_device_exit_batch (net/core/dev.c:11592) [ 2533.721916] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2533.722112] ? process_one_work (kernel/workqueue.c:2606) [ 2533.722350] process_one_work (kernel/workqueue.c:2633) [ 2533.722569] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2533.722774] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2533.723004] kthread (kernel/kthread.c:388) [ 2533.723177] ? __pfx_kthread (kernel/kthread.c:341) [ 2533.723379] ret_from_fork (arch/x86/kernel/process.c:147) [ 2533.723575] ? __pfx_kthread (kernel/kthread.c:341) [ 2533.723779] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2602.048621] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2591049 jiffies s: 4157 root: 0x8/. | [ 2602.049236] rcu: blocking rcu_node structures (internal RCU debug): | [ 2602.049643] Sending NMI from CPU 2 to CPUs 3: | [ 2602.049886] NMI backtrace for cpu 3 [ 2602.049894] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2602.049896] Workqueue: netns cleanup_net [ 2602.049905] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2602.049913] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2602.049914] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2602.049916] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2602.049917] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2602.049918] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2602.049919] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2602.049921] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2602.049923] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2602.049925] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2602.049925] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2602.049926] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2602.049927] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2602.049928] PKRU: 55555554 [ 2602.049929] Call Trace: [ 2602.049932] [ 2602.049935] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 2602.049940] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 2602.049944] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 2602.049948] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 2602.049952] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 2602.049954] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 2602.049959] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2602.049962] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2602.049964] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2602.049966] [ 2602.049967] [ 2602.049968] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2602.049973] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2602.049975] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2602.049980] ? up_read (kernel/locking/rwsem.c:1622) [ 2602.049983] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2602.049988] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2602.049990] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2602.049992] __tcf_action_put (net/sched/act_api.c:377) [ 2602.049997] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2602.050000] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2602.050002] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2602.050008] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2602.050014] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2602.050018] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2602.050022] tcf_chain_flush (net/sched/cls_api.c:737) [ 2602.050024] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2602.050026] tcf_block_put (net/sched/cls_api.c:1543) [ 2602.050029] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2602.050033] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2602.050038] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2602.050040] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2602.050044] default_device_exit_batch (net/core/dev.c:11592) [ 2602.050047] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2602.050050] ? process_one_work (kernel/workqueue.c:2606) [ 2602.050054] process_one_work (kernel/workqueue.c:2633) [ 2602.050057] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2602.050059] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2602.050061] kthread (kernel/kthread.c:388) [ 2602.050064] ? __pfx_kthread (kernel/kthread.c:341) [ 2602.050065] ret_from_fork (arch/x86/kernel/process.c:147) [ 2602.050069] ? __pfx_kthread (kernel/kthread.c:341) [ 2602.050070] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2602.050075] | [ 2611.727609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2611.727932] rcu: 3-....: (2599754 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=595451 | [ 2611.728448] rcu: (t=2600758 jiffies g=7477 q=1775 ncpus=4) [ 2611.729191] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2611.729786] Workqueue: netns cleanup_net [ 2611.730000] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2611.730295] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2611.731287] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2611.731566] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2611.731941] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2611.732316] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2611.732691] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2611.733065] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2611.733438] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2611.733876] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2611.734182] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2611.734556] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2611.734931] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2611.735305] PKRU: 55555554 [ 2611.735452] Call Trace: [ 2611.735587] [ 2611.735705] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2611.735946] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2611.736194] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 2611.736406] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 2611.736615] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 2611.736821] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2611.737061] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2611.737297] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2611.737581] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2611.737800] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2611.738057] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2611.738297] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2611.738527] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2611.738808] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2611.739073] [ 2611.739191] [ 2611.739309] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2611.739594] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2611.739856] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2611.740063] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2611.740262] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2611.740528] ? up_read (kernel/locking/rwsem.c:1622) [ 2611.740710] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2611.740950] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2611.741215] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2611.741471] __tcf_action_put (net/sched/act_api.c:377) [ 2611.741683] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2611.741898] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2611.742105] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2611.742386] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2611.742641] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2611.742889] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2611.743101] tcf_chain_flush (net/sched/cls_api.c:737) [ 2611.743309] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2611.743518] tcf_block_put (net/sched/cls_api.c:1543) [ 2611.743720] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2611.743960] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2611.744171] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2611.744365] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2611.744658] default_device_exit_batch (net/core/dev.c:11592) [ 2611.744916] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2611.745113] ? process_one_work (kernel/workqueue.c:2606) [ 2611.745337] process_one_work (kernel/workqueue.c:2633) [ 2611.745555] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2611.745760] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2611.745989] kthread (kernel/kthread.c:388) [ 2611.746164] ? __pfx_kthread (kernel/kthread.c:341) [ 2611.746367] ret_from_fork (arch/x86/kernel/process.c:147) [ 2611.746566] ? __pfx_kthread (kernel/kthread.c:341) [ 2611.746769] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2611.746988] | [ 2689.750609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2689.750951] rcu: 3-....: (2677759 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=613943 | [ 2689.751471] rcu: (t=2678781 jiffies g=7477 q=1775 ncpus=4) [ 2689.752225] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2689.752822] Workqueue: netns cleanup_net [ 2689.753047] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2689.753347] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2689.754313] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2689.754592] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2689.754969] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2689.755344] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2689.755733] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2689.756109] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2689.756486] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2689.756910] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2689.757216] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2689.757592] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2689.757967] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2689.758341] PKRU: 55555554 [ 2689.758491] Call Trace: [ 2689.758632] [ 2689.758755] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2689.758996] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2689.759241] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 2689.759456] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 2689.759668] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 2689.759875] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2689.760116] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2689.760351] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2689.760635] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2689.760856] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2689.761113] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2689.761353] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2689.761585] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2689.761865] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2689.762130] [ 2689.762250] [ 2689.762369] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2689.762654] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2689.762917] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2689.763127] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2689.763333] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2689.763601] ? up_read (kernel/locking/rwsem.c:1622) [ 2689.763781] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2689.764023] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2689.764287] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2689.764543] __tcf_action_put (net/sched/act_api.c:377) [ 2689.764755] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2689.764975] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2689.765183] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2689.765465] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2689.765723] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2689.765970] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2689.766182] tcf_chain_flush (net/sched/cls_api.c:737) [ 2689.766389] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2689.766598] tcf_block_put (net/sched/cls_api.c:1543) [ 2689.766795] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2689.767037] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2689.767247] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2689.767444] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2689.767737] default_device_exit_batch (net/core/dev.c:11592) [ 2689.767995] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2689.768191] ? process_one_work (kernel/workqueue.c:2606) [ 2689.768420] process_one_work (kernel/workqueue.c:2633) [ 2689.768638] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2689.768844] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2689.769074] kthread (kernel/kthread.c:388) [ 2689.769250] ? __pfx_kthread (kernel/kthread.c:341) [ 2689.769453] ret_from_fork (arch/x86/kernel/process.c:147) [ 2689.769651] ? __pfx_kthread (kernel/kthread.c:341) [ 2689.769854] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2700.352623] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2689353 jiffies s: 4157 root: 0x8/. | [ 2700.353243] rcu: blocking rcu_node structures (internal RCU debug): | [ 2700.353577] Sending NMI from CPU 2 to CPUs 3: | [ 2700.353831] NMI backtrace for cpu 3 [ 2700.353837] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2700.353839] Workqueue: netns cleanup_net [ 2700.353845] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2700.353850] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2700.353852] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2700.353853] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2700.353855] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2700.353856] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2700.353857] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2700.353858] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2700.353861] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2700.353862] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2700.353863] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2700.353864] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2700.353865] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2700.353865] PKRU: 55555554 [ 2700.353866] Call Trace: [ 2700.353868] [ 2700.353870] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 2700.353875] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 2700.353880] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 2700.353883] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 2700.353888] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 2700.353890] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 2700.353894] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2700.353896] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2700.353899] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2700.353901] [ 2700.353901] [ 2700.353902] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2700.353906] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2700.353908] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2700.353911] ? up_read (kernel/locking/rwsem.c:1622) [ 2700.353914] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2700.353917] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2700.353919] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2700.353921] __tcf_action_put (net/sched/act_api.c:377) [ 2700.353925] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2700.353927] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2700.353930] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2700.353936] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2700.353941] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2700.353946] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2700.353949] tcf_chain_flush (net/sched/cls_api.c:737) [ 2700.353950] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2700.353952] tcf_block_put (net/sched/cls_api.c:1543) [ 2700.353953] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2700.353957] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2700.353960] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2700.353963] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2700.353966] default_device_exit_batch (net/core/dev.c:11592) [ 2700.353968] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2700.353971] ? process_one_work (kernel/workqueue.c:2606) [ 2700.353974] process_one_work (kernel/workqueue.c:2633) [ 2700.353976] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2700.353979] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2700.353980] kthread (kernel/kthread.c:388) [ 2700.353982] ? __pfx_kthread (kernel/kthread.c:341) [ 2700.353984] ret_from_fork (arch/x86/kernel/process.c:147) [ 2700.353988] ? __pfx_kthread (kernel/kthread.c:341) [ 2700.353989] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2700.353993] | [ 2767.773609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2767.773963] rcu: 3-....: (2755749 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=632534 | [ 2767.774479] rcu: (t=2756804 jiffies g=7477 q=1775 ncpus=4) [ 2767.775226] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2767.775820] Workqueue: netns cleanup_net [ 2767.776039] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 2767.776333] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 <8b> 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: 90 nop f: 90 nop 10: 90 nop 11: f3 0f 1e fa endbr64 15: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 1a: 41 55 push %r13 1c: 41 54 push %r12 1e: 55 push %rbp 1f: 48 89 fd mov %rdi,%rbp 22: 53 push %rbx 23: 66 90 xchg %ax,%ax 25: ba 01 00 00 00 mov $0x1,%edx 2a:* 8b 45 00 mov 0x0(%rbp),%eax <-- trapping instruction 2d: 85 c0 test %eax,%eax 2f: 75 14 jne 0x45 31: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 36: 85 c0 test %eax,%eax 38: 75 f0 jne 0x2a 3a: 5b pop %rbx 3b: 5d pop %rbp 3c: 41 5c pop %r12 3e: 41 5d pop %r13 Code starting with the faulting instruction =========================================== 0: 8b 45 00 mov 0x0(%rbp),%eax 3: 85 c0 test %eax,%eax 5: 75 14 jne 0x1b 7: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) c: 85 c0 test %eax,%eax e: 75 f0 jne 0x0 10: 5b pop %rbx 11: 5d pop %rbp 12: 41 5c pop %r12 14: 41 5d pop %r13 [ 2767.777294] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2767.777571] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2767.777944] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2767.778318] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2767.778692] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2767.779063] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2767.779440] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2767.779863] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2767.780166] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2767.780541] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2767.780915] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2767.781296] PKRU: 55555554 [ 2767.781444] Call Trace: [ 2767.781583] [ 2767.781701] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2767.781944] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2767.782186] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 2767.782400] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 2767.782609] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 2767.782814] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2767.783060] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2767.783294] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2767.783578] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2767.783797] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2767.784053] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2767.784293] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2767.784523] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2767.784801] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2767.785067] [ 2767.785184] [ 2767.785303] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2767.785587] ? queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 2767.785849] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2767.786058] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2767.786256] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2767.786524] ? up_read (kernel/locking/rwsem.c:1622) [ 2767.786704] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2767.786943] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2767.787206] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2767.787462] __tcf_action_put (net/sched/act_api.c:377) [ 2767.787673] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2767.787890] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2767.788097] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2767.788379] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2767.788635] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2767.788881] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2767.789093] tcf_chain_flush (net/sched/cls_api.c:737) [ 2767.789299] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2767.789507] tcf_block_put (net/sched/cls_api.c:1543) [ 2767.789703] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2767.789944] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2767.790152] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2767.790347] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2767.790638] default_device_exit_batch (net/core/dev.c:11592) [ 2767.790895] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2767.791092] ? process_one_work (kernel/workqueue.c:2606) [ 2767.791316] process_one_work (kernel/workqueue.c:2633) [ 2767.791532] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2767.791737] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2767.791968] kthread (kernel/kthread.c:388) [ 2767.792142] ? __pfx_kthread (kernel/kthread.c:341) [ 2767.792344] ret_from_fork (arch/x86/kernel/process.c:147) [ 2767.792541] ? __pfx_kthread (kernel/kthread.c:341) [ 2767.792743] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2798.656631] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2787657 jiffies s: 4157 root: 0x8/. | [ 2798.657513] rcu: blocking rcu_node structures (internal RCU debug): | [ 2798.658007] Sending NMI from CPU 2 to CPUs 3: | [ 2798.658364] NMI backtrace for cpu 3 [ 2798.658370] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2798.658372] Workqueue: netns cleanup_net [ 2798.658381] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2798.658387] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2798.658389] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2798.658390] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2798.658392] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2798.658393] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2798.658394] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2798.658395] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2798.658398] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2798.658399] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2798.658400] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2798.658401] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2798.658401] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2798.658402] PKRU: 55555554 [ 2798.658403] Call Trace: [ 2798.658405] [ 2798.658407] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 2798.658413] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 2798.658418] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 2798.658421] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 2798.658426] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 2798.658428] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 2798.658433] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2798.658435] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2798.658437] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2798.658440] [ 2798.658440] [ 2798.658441] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2798.658447] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2798.658449] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2798.658454] ? up_read (kernel/locking/rwsem.c:1622) [ 2798.658457] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2798.658462] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2798.658464] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2798.658466] __tcf_action_put (net/sched/act_api.c:377) [ 2798.658471] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2798.658474] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2798.658476] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2798.658483] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2798.658489] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2798.658494] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2798.658497] tcf_chain_flush (net/sched/cls_api.c:737) [ 2798.658499] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2798.658501] tcf_block_put (net/sched/cls_api.c:1543) [ 2798.658505] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2798.658509] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2798.658514] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2798.658517] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2798.658522] default_device_exit_batch (net/core/dev.c:11592) [ 2798.658525] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2798.658528] ? process_one_work (kernel/workqueue.c:2606) [ 2798.658532] process_one_work (kernel/workqueue.c:2633) [ 2798.658535] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2798.658537] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2798.658539] kthread (kernel/kthread.c:388) [ 2798.658542] ? __pfx_kthread (kernel/kthread.c:341) [ 2798.658544] ret_from_fork (arch/x86/kernel/process.c:147) [ 2798.658548] ? __pfx_kthread (kernel/kthread.c:341) [ 2798.658549] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2798.658554] | [ 2845.796609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2845.796950] rcu: 3-....: (2833739 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=650815 | [ 2845.797461] rcu: (t=2834827 jiffies g=7477 q=1775 ncpus=4) [ 2845.798194] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2845.798790] Workqueue: netns cleanup_net [ 2845.799003] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2845.799290] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2845.800238] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2845.800514] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2845.800885] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2845.801250] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2845.801624] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2845.802068] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2845.802520] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2845.803021] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2845.803381] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2845.803826] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2845.804269] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2845.804711] PKRU: 55555554 [ 2845.804886] Call Trace: [ 2845.805048] [ 2845.805185] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2845.805468] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2845.805757] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 2845.806009] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 2845.806257] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 2845.806502] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2845.806784] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2845.807062] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2845.807398] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2845.807660] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2845.807963] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2845.808247] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2845.808521] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2845.808850] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2845.809162] [ 2845.809302] [ 2845.809443] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2845.809778] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2845.810086] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2845.810338] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2845.810573] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2845.810890] ? up_read (kernel/locking/rwsem.c:1622) [ 2845.811100] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2845.811386] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2845.811698] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2845.812001] __tcf_action_put (net/sched/act_api.c:377) [ 2845.812248] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2845.812512] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2845.812758] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2845.813092] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2845.813396] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2845.813692] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2845.813943] tcf_chain_flush (net/sched/cls_api.c:737) [ 2845.814187] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2845.814432] tcf_block_put (net/sched/cls_api.c:1543) [ 2845.814664] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2845.814964] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2845.815211] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2845.815443] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2845.815785] default_device_exit_batch (net/core/dev.c:11592) [ 2845.816088] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2845.816321] ? process_one_work (kernel/workqueue.c:2606) [ 2845.816590] process_one_work (kernel/workqueue.c:2633) [ 2845.816846] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2845.817086] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2845.817360] kthread (kernel/kthread.c:388) [ 2845.817566] ? __pfx_kthread (kernel/kthread.c:341) [ 2845.817807] ret_from_fork (arch/x86/kernel/process.c:147) [ 2845.818039] ? __pfx_kthread (kernel/kthread.c:341) [ 2845.818277] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2896.960621] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2885961 jiffies s: 4157 root: 0x8/. | [ 2896.961277] rcu: blocking rcu_node structures (internal RCU debug): | [ 2896.961614] Sending NMI from CPU 2 to CPUs 3: | [ 2896.961869] NMI backtrace for cpu 3 [ 2896.961876] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2896.961878] Workqueue: netns cleanup_net [ 2896.961886] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2896.961892] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2896.961894] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2896.961895] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2896.961897] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2896.961898] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2896.961899] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2896.961900] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2896.961903] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2896.961904] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2896.961905] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2896.961906] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2896.961907] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2896.961907] PKRU: 55555554 [ 2896.961908] Call Trace: [ 2896.961911] [ 2896.961914] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 2896.961919] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 2896.961924] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 2896.961926] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 2896.961931] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 2896.961933] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 2896.961938] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2896.961941] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2896.961943] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2896.961945] [ 2896.961946] [ 2896.961946] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2896.961952] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2896.961954] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2896.961959] ? up_read (kernel/locking/rwsem.c:1622) [ 2896.961962] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2896.961967] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2896.961969] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2896.961971] __tcf_action_put (net/sched/act_api.c:377) [ 2896.961976] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2896.961979] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2896.961982] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2896.961988] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2896.961995] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2896.962000] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2896.962003] tcf_chain_flush (net/sched/cls_api.c:737) [ 2896.962005] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2896.962007] tcf_block_put (net/sched/cls_api.c:1543) [ 2896.962009] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2896.962014] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2896.962019] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2896.962022] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2896.962027] default_device_exit_batch (net/core/dev.c:11592) [ 2896.962030] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2896.962033] ? process_one_work (kernel/workqueue.c:2606) [ 2896.962037] process_one_work (kernel/workqueue.c:2633) [ 2896.962040] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2896.962043] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2896.962044] kthread (kernel/kthread.c:388) [ 2896.962047] ? __pfx_kthread (kernel/kthread.c:341) [ 2896.962049] ret_from_fork (arch/x86/kernel/process.c:147) [ 2896.962053] ? __pfx_kthread (kernel/kthread.c:341) [ 2896.962054] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2896.962059] | [ 2923.822609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2923.822952] rcu: 3-....: (2911729 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=668812 | [ 2923.823471] rcu: (t=2912853 jiffies g=7477 q=1775 ncpus=4) [ 2923.824220] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2923.824821] Workqueue: netns cleanup_net [ 2923.825042] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2923.825336] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2923.826305] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2923.826582] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2923.826961] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2923.827333] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2923.827711] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2923.828087] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2923.828463] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2923.828895] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2923.829201] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2923.829578] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2923.829957] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2923.830329] PKRU: 55555554 [ 2923.830479] Call Trace: [ 2923.830616] [ 2923.830738] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2923.830977] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2923.831226] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 2923.831439] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 2923.831648] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 2923.831862] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2923.832109] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2923.832346] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2923.832628] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2923.832856] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2923.833111] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2923.833351] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2923.833584] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2923.833869] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2923.834133] [ 2923.834252] [ 2923.834372] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2923.834656] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2923.834923] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2923.835131] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2923.835330] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2923.835598] ? up_read (kernel/locking/rwsem.c:1622) [ 2923.835782] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2923.836025] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2923.836289] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2923.836545] __tcf_action_put (net/sched/act_api.c:377) [ 2923.836761] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2923.836976] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2923.837184] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2923.837467] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2923.837729] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2923.837973] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2923.838187] tcf_chain_flush (net/sched/cls_api.c:737) [ 2923.838394] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2923.838600] tcf_block_put (net/sched/cls_api.c:1543) [ 2923.838800] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2923.839043] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2923.839252] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2923.839448] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2923.839741] default_device_exit_batch (net/core/dev.c:11592) [ 2923.839998] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2923.840193] ? process_one_work (kernel/workqueue.c:2606) [ 2923.840420] process_one_work (kernel/workqueue.c:2633) [ 2923.840636] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2923.840845] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2923.841077] kthread (kernel/kthread.c:388) [ 2923.841249] ? __pfx_kthread (kernel/kthread.c:341) [ 2923.841452] ret_from_fork (arch/x86/kernel/process.c:147) [ 2923.841648] ? __pfx_kthread (kernel/kthread.c:341) [ 2923.841859] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2995.264646] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2984265 jiffies s: 4157 root: 0x8/. | [ 2995.265482] rcu: blocking rcu_node structures (internal RCU debug): | [ 2995.265949] Sending NMI from CPU 2 to CPUs 3: | [ 2995.266289] NMI backtrace for cpu 3 [ 2995.266297] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2995.266301] Workqueue: netns cleanup_net [ 2995.266310] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2995.266317] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2995.266319] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 2995.266320] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 2995.266321] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 2995.266322] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 2995.266323] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 2995.266324] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 2995.266327] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 2995.266328] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2995.266329] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 2995.266330] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2995.266330] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2995.266331] PKRU: 55555554 [ 2995.266332] Call Trace: [ 2995.266339] [ 2995.266345] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 2995.266350] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 2995.266355] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 2995.266358] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 2995.266362] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 2995.266364] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 2995.266369] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2995.266371] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2995.266374] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2995.266376] [ 2995.266377] [ 2995.266378] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2995.266383] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2995.266385] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2995.266390] ? up_read (kernel/locking/rwsem.c:1622) [ 2995.266394] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2995.266399] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2995.266401] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2995.266403] __tcf_action_put (net/sched/act_api.c:377) [ 2995.266409] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2995.266411] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2995.266413] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2995.266420] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2995.266426] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2995.266432] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2995.266435] tcf_chain_flush (net/sched/cls_api.c:737) [ 2995.266437] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2995.266439] tcf_block_put (net/sched/cls_api.c:1543) [ 2995.266442] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2995.266446] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2995.266451] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2995.266454] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 2995.266459] default_device_exit_batch (net/core/dev.c:11592) [ 2995.266462] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2995.266465] ? process_one_work (kernel/workqueue.c:2606) [ 2995.266470] process_one_work (kernel/workqueue.c:2633) [ 2995.266473] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2995.266476] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2995.266478] kthread (kernel/kthread.c:388) [ 2995.266480] ? __pfx_kthread (kernel/kthread.c:341) [ 2995.266482] ret_from_fork (arch/x86/kernel/process.c:147) [ 2995.266486] ? __pfx_kthread (kernel/kthread.c:341) [ 2995.266488] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2995.266494] | [ 3001.845609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3001.845960] rcu: 3-....: (2989719 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=686764 | [ 3001.846479] rcu: (t=2990876 jiffies g=7477 q=1775 ncpus=4) [ 3001.847222] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3001.847817] Workqueue: netns cleanup_net [ 3001.848034] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3001.848331] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3001.849294] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3001.849573] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3001.849946] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3001.850321] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3001.850696] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3001.851071] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3001.851444] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3001.851874] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3001.852178] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3001.852550] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3001.852924] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3001.853299] PKRU: 55555554 [ 3001.853447] Call Trace: [ 3001.853584] [ 3001.853700] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3001.853941] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3001.854186] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 3001.854400] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 3001.854609] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 3001.854816] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3001.855056] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3001.855291] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3001.855577] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3001.855796] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3001.856052] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3001.856293] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3001.856523] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3001.856807] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3001.857073] [ 3001.857190] [ 3001.857307] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3001.857594] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3001.857856] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3001.858064] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3001.858263] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3001.858531] ? up_read (kernel/locking/rwsem.c:1622) [ 3001.858710] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3001.858952] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3001.859217] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3001.859474] __tcf_action_put (net/sched/act_api.c:377) [ 3001.859684] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3001.859899] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3001.860109] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3001.860390] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3001.860651] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3001.860898] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3001.861109] tcf_chain_flush (net/sched/cls_api.c:737) [ 3001.861316] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3001.861524] tcf_block_put (net/sched/cls_api.c:1543) [ 3001.861725] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3001.861964] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3001.862176] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3001.862371] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3001.862659] default_device_exit_batch (net/core/dev.c:11592) [ 3001.862915] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3001.863111] ? process_one_work (kernel/workqueue.c:2606) [ 3001.863337] process_one_work (kernel/workqueue.c:2633) [ 3001.863555] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3001.863759] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3001.863987] kthread (kernel/kthread.c:388) [ 3001.864163] ? __pfx_kthread (kernel/kthread.c:341) [ 3001.864366] ret_from_fork (arch/x86/kernel/process.c:147) [ 3001.864562] ? __pfx_kthread (kernel/kthread.c:341) [ 3001.864766] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3001.864982] | [ 3079.868608] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3079.868960] rcu: 3-....: (3067724 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=704673 | [ 3079.869479] rcu: (t=3068899 jiffies g=7477 q=1775 ncpus=4) [ 3079.870228] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3079.870823] Workqueue: netns cleanup_net [ 3079.871046] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3079.871347] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3079.872311] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3079.872589] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3079.872965] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3079.873338] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3079.873714] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3079.874089] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3079.874465] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3079.874889] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3079.875193] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3079.875568] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3079.875942] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3079.876318] PKRU: 55555554 [ 3079.876468] Call Trace: [ 3079.876608] [ 3079.876723] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3079.876963] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3079.877208] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 3079.877420] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 3079.877629] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 3079.877835] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3079.878076] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3079.878312] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3079.878596] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3079.878814] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3079.879070] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3079.879312] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3079.879543] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3079.879821] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3079.880087] [ 3079.880206] [ 3079.880324] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3079.880607] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3079.880870] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3079.881077] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3079.881276] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3079.881546] ? up_read (kernel/locking/rwsem.c:1622) [ 3079.881726] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3079.881969] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3079.882233] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3079.882489] __tcf_action_put (net/sched/act_api.c:377) [ 3079.882698] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3079.882913] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3079.883120] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3079.883407] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3079.883664] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3079.883910] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3079.884123] tcf_chain_flush (net/sched/cls_api.c:737) [ 3079.884331] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3079.884539] tcf_block_put (net/sched/cls_api.c:1543) [ 3079.884736] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3079.884978] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3079.885189] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3079.885387] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3079.885675] default_device_exit_batch (net/core/dev.c:11592) [ 3079.885933] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3079.886130] ? process_one_work (kernel/workqueue.c:2606) [ 3079.886364] process_one_work (kernel/workqueue.c:2633) [ 3079.886583] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3079.886787] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3079.887018] kthread (kernel/kthread.c:388) [ 3079.887193] ? __pfx_kthread (kernel/kthread.c:341) [ 3079.887396] ret_from_fork (arch/x86/kernel/process.c:147) [ 3079.887592] ? __pfx_kthread (kernel/kthread.c:341) [ 3079.887796] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3093.568620] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 3082569 jiffies s: 4157 root: 0x8/. | [ 3093.569233] rcu: blocking rcu_node structures (internal RCU debug): | [ 3093.569571] Sending NMI from CPU 2 to CPUs 3: | [ 3093.569821] NMI backtrace for cpu 3 [ 3093.569827] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3093.569828] Workqueue: netns cleanup_net [ 3093.569833] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 3093.569838] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 <85> c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: f3 0f 1e fa endbr64 12: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 17: 41 55 push %r13 19: 41 54 push %r12 1b: 55 push %rbp 1c: 48 89 fd mov %rdi,%rbp 1f: 53 push %rbx 20: 66 90 xchg %ax,%ax 22: ba 01 00 00 00 mov $0x1,%edx 27: 8b 45 00 mov 0x0(%rbp),%eax 2a:* 85 c0 test %eax,%eax <-- trapping instruction 2c: 75 14 jne 0x42 2e: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 33: 85 c0 test %eax,%eax 35: 75 f0 jne 0x27 37: 5b pop %rbx 38: 5d pop %rbp 39: 41 5c pop %r12 3b: 41 5d pop %r13 3d: c3 ret 3e: cc int3 3f: cc int3 Code starting with the faulting instruction =========================================== 0: 85 c0 test %eax,%eax 2: 75 14 jne 0x18 4: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 9: 85 c0 test %eax,%eax b: 75 f0 jne 0xfffffffffffffffd d: 5b pop %rbx e: 5d pop %rbp f: 41 5c pop %r12 11: 41 5d pop %r13 13: c3 ret 14: cc int3 15: cc int3 [ 3093.569839] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3093.569841] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3093.569842] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3093.569843] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3093.569844] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3093.569845] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3093.569847] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3093.569849] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3093.569849] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3093.569850] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3093.569851] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3093.569852] PKRU: 55555554 [ 3093.569852] Call Trace: [ 3093.569855] [ 3093.569856] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 3093.569861] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 3093.569865] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 3093.569868] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 3093.569873] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 3093.569875] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 3093.569879] ? queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 3093.569881] ? queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 3093.569883] ? queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 3093.569885] [ 3093.569886] [ 3093.569887] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3093.569891] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3093.569892] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3093.569895] ? up_read (kernel/locking/rwsem.c:1622) [ 3093.569898] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3093.569902] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3093.569904] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3093.569906] __tcf_action_put (net/sched/act_api.c:377) [ 3093.569909] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3093.569911] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3093.569914] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3093.569920] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3093.569925] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3093.569930] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3093.569933] tcf_chain_flush (net/sched/cls_api.c:737) [ 3093.569934] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3093.569936] tcf_block_put (net/sched/cls_api.c:1543) [ 3093.569938] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3093.569942] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3093.569945] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3093.569947] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3093.569951] default_device_exit_batch (net/core/dev.c:11592) [ 3093.569954] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3093.569956] ? process_one_work (kernel/workqueue.c:2606) [ 3093.569960] process_one_work (kernel/workqueue.c:2633) [ 3093.569963] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3093.569965] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3093.569967] kthread (kernel/kthread.c:388) [ 3093.569969] ? __pfx_kthread (kernel/kthread.c:341) [ 3093.569971] ret_from_fork (arch/x86/kernel/process.c:147) [ 3093.569974] ? __pfx_kthread (kernel/kthread.c:341) [ 3093.569976] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3093.569981] | [ 3157.891609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3157.891954] rcu: 3-....: (3145714 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=722405 | [ 3157.892475] rcu: (t=3146922 jiffies g=7477 q=1775 ncpus=4) [ 3157.893231] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3157.893831] Workqueue: netns cleanup_net [ 3157.894053] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3157.894350] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3157.895319] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3157.895596] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3157.895973] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3157.896347] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3157.896725] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3157.897102] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3157.897480] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3157.897905] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3157.898212] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3157.898588] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3157.898963] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3157.899339] PKRU: 55555554 [ 3157.899489] Call Trace: [ 3157.899630] [ 3157.899753] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3157.899993] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3157.900239] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 3157.900452] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 3157.900664] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 3157.900872] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3157.901116] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3157.901352] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3157.901637] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3157.901859] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3157.902117] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3157.902355] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3157.902588] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3157.902868] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3157.903133] [ 3157.903252] [ 3157.903371] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3157.903658] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3157.903919] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3157.904129] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3157.904329] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3157.904598] ? up_read (kernel/locking/rwsem.c:1622) [ 3157.904776] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3157.905020] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3157.905285] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3157.905542] __tcf_action_put (net/sched/act_api.c:377) [ 3157.905756] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3157.905976] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3157.906185] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3157.906469] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3157.906727] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3157.906976] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3157.907189] tcf_chain_flush (net/sched/cls_api.c:737) [ 3157.907397] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3157.907604] tcf_block_put (net/sched/cls_api.c:1543) [ 3157.907802] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3157.908046] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3157.908256] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3157.908452] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3157.908745] default_device_exit_batch (net/core/dev.c:11592) [ 3157.909003] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3157.909200] ? process_one_work (kernel/workqueue.c:2606) [ 3157.909428] process_one_work (kernel/workqueue.c:2633) [ 3157.909646] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3157.909851] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3157.910082] kthread (kernel/kthread.c:388) [ 3157.910258] ? __pfx_kthread (kernel/kthread.c:341) [ 3157.910461] ret_from_fork (arch/x86/kernel/process.c:147) [ 3157.910660] ? __pfx_kthread (kernel/kthread.c:341) [ 3157.910867] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3191.872622] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 3180873 jiffies s: 4157 root: 0x8/. | [ 3191.873219] rcu: blocking rcu_node structures (internal RCU debug): | [ 3191.873542] Sending NMI from CPU 2 to CPUs 3: | [ 3191.873791] NMI backtrace for cpu 3 [ 3191.873797] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3191.873799] Workqueue: netns cleanup_net [ 3191.873806] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3191.873812] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3191.873814] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3191.873815] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3191.873816] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3191.873817] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3191.873818] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3191.873819] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3191.873822] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3191.873823] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3191.873824] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3191.873825] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3191.873826] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3191.873827] PKRU: 55555554 [ 3191.873827] Call Trace: [ 3191.873830] [ 3191.873832] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 3191.873838] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 3191.873843] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 3191.873846] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 3191.873849] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 3191.873852] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 3191.873856] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3191.873859] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3191.873861] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3191.873863] [ 3191.873864] [ 3191.873864] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3191.873869] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3191.873871] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3191.873875] ? up_read (kernel/locking/rwsem.c:1622) [ 3191.873879] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3191.873883] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3191.873885] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3191.873887] __tcf_action_put (net/sched/act_api.c:377) [ 3191.873892] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3191.873894] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3191.873896] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3191.873902] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3191.873908] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3191.873912] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3191.873915] tcf_chain_flush (net/sched/cls_api.c:737) [ 3191.873917] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3191.873919] tcf_block_put (net/sched/cls_api.c:1543) [ 3191.873922] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3191.873927] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3191.873931] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3191.873935] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3191.873938] default_device_exit_batch (net/core/dev.c:11592) [ 3191.873941] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3191.873944] ? process_one_work (kernel/workqueue.c:2606) [ 3191.873948] process_one_work (kernel/workqueue.c:2633) [ 3191.873951] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3191.873953] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3191.873955] kthread (kernel/kthread.c:388) [ 3191.873958] ? __pfx_kthread (kernel/kthread.c:341) [ 3191.873959] ret_from_fork (arch/x86/kernel/process.c:147) [ 3191.873963] ? __pfx_kthread (kernel/kthread.c:341) [ 3191.873965] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3191.873969] | [ 3235.914609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3235.914944] rcu: 3-....: (3223704 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=739743 | [ 3235.915463] rcu: (t=3224945 jiffies g=7477 q=1775 ncpus=4) [ 3235.916207] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3235.916803] Workqueue: netns cleanup_net [ 3235.917023] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3235.917319] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3235.918287] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3235.918566] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3235.918961] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3235.919336] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3235.919711] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3235.920086] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3235.920464] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3235.920895] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3235.921201] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3235.921577] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3235.921951] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3235.922325] PKRU: 55555554 [ 3235.922475] Call Trace: [ 3235.922612] [ 3235.922728] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3235.922968] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3235.923213] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 3235.923425] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 3235.923634] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 3235.923842] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3235.924081] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3235.924317] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3235.924600] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3235.924820] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3235.925076] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3235.925317] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3235.925552] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3235.925831] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3235.926097] [ 3235.926216] [ 3235.926336] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3235.926621] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3235.926883] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3235.927091] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3235.927291] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3235.927558] ? up_read (kernel/locking/rwsem.c:1622) [ 3235.927739] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3235.927985] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3235.928250] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3235.928508] __tcf_action_put (net/sched/act_api.c:377) [ 3235.928720] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3235.928935] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3235.929146] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3235.929430] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3235.929690] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3235.929936] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3235.930151] tcf_chain_flush (net/sched/cls_api.c:737) [ 3235.930359] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3235.930573] tcf_block_put (net/sched/cls_api.c:1543) [ 3235.930770] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3235.931014] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3235.931225] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3235.931421] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3235.931713] default_device_exit_batch (net/core/dev.c:11592) [ 3235.931972] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3235.932168] ? process_one_work (kernel/workqueue.c:2606) [ 3235.932398] process_one_work (kernel/workqueue.c:2633) [ 3235.932616] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3235.932820] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3235.933053] kthread (kernel/kthread.c:388) [ 3235.933226] ? __pfx_kthread (kernel/kthread.c:341) [ 3235.933430] ret_from_fork (arch/x86/kernel/process.c:147) [ 3235.933628] ? __pfx_kthread (kernel/kthread.c:341) [ 3235.933832] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3290.176623] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 3279177 jiffies s: 4157 root: 0x8/. | [ 3290.177244] rcu: blocking rcu_node structures (internal RCU debug): | [ 3290.177572] Sending NMI from CPU 2 to CPUs 3: | [ 3290.177823] NMI backtrace for cpu 3 [ 3290.177829] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3290.177832] Workqueue: netns cleanup_net [ 3290.177838] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3290.177843] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3290.177845] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3290.177846] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3290.177847] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3290.177848] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3290.177850] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3290.177850] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3290.177853] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3290.177854] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3290.177855] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3290.177856] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3290.177857] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3290.177857] PKRU: 55555554 [ 3290.177858] Call Trace: [ 3290.177861] [ 3290.177863] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 3290.177868] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 3290.177872] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 3290.177875] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 3290.177879] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 3290.177881] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 3290.177886] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3290.177888] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3290.177891] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3290.177893] [ 3290.177893] [ 3290.177894] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3290.177899] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3290.177901] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3290.177905] ? up_read (kernel/locking/rwsem.c:1622) [ 3290.177908] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3290.177911] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3290.177913] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3290.177916] __tcf_action_put (net/sched/act_api.c:377) [ 3290.177921] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3290.177923] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3290.177925] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3290.177932] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3290.177937] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3290.177942] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3290.177945] tcf_chain_flush (net/sched/cls_api.c:737) [ 3290.177947] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3290.177949] tcf_block_put (net/sched/cls_api.c:1543) [ 3290.177951] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3290.177955] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3290.177960] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3290.177962] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3290.177966] default_device_exit_batch (net/core/dev.c:11592) [ 3290.177969] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3290.177972] ? process_one_work (kernel/workqueue.c:2606) [ 3290.177976] process_one_work (kernel/workqueue.c:2633) [ 3290.177979] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3290.177982] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3290.177984] kthread (kernel/kthread.c:388) [ 3290.177986] ? __pfx_kthread (kernel/kthread.c:341) [ 3290.177988] ret_from_fork (arch/x86/kernel/process.c:147) [ 3290.177992] ? __pfx_kthread (kernel/kthread.c:341) [ 3290.177993] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3290.177997] | [ 3313.937609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3313.937944] rcu: 3-....: (3301694 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=757105 | [ 3313.938462] rcu: (t=3302968 jiffies g=7477 q=1775 ncpus=4) [ 3313.939209] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3313.939803] Workqueue: netns cleanup_net [ 3313.940020] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 3313.940319] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 <8b> 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: 90 nop f: 90 nop 10: 90 nop 11: f3 0f 1e fa endbr64 15: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 1a: 41 55 push %r13 1c: 41 54 push %r12 1e: 55 push %rbp 1f: 48 89 fd mov %rdi,%rbp 22: 53 push %rbx 23: 66 90 xchg %ax,%ax 25: ba 01 00 00 00 mov $0x1,%edx 2a:* 8b 45 00 mov 0x0(%rbp),%eax <-- trapping instruction 2d: 85 c0 test %eax,%eax 2f: 75 14 jne 0x45 31: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 36: 85 c0 test %eax,%eax 38: 75 f0 jne 0x2a 3a: 5b pop %rbx 3b: 5d pop %rbp 3c: 41 5c pop %r12 3e: 41 5d pop %r13 Code starting with the faulting instruction =========================================== 0: 8b 45 00 mov 0x0(%rbp),%eax 3: 85 c0 test %eax,%eax 5: 75 14 jne 0x1b 7: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) c: 85 c0 test %eax,%eax e: 75 f0 jne 0x0 10: 5b pop %rbx 11: 5d pop %rbp 12: 41 5c pop %r12 14: 41 5d pop %r13 [ 3313.941283] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3313.941560] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3313.941936] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3313.942309] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3313.942683] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3313.943057] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3313.943432] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3313.943857] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3313.944160] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3313.944538] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3313.944927] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3313.945314] PKRU: 55555554 [ 3313.945467] Call Trace: [ 3313.945608] [ 3313.945731] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3313.945977] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3313.946228] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 3313.946449] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 3313.946665] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 3313.946880] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3313.947127] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3313.947363] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3313.947647] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3313.947866] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3313.948123] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3313.948361] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3313.948592] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3313.948870] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3313.949134] [ 3313.949252] [ 3313.949370] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3313.949654] ? queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 3313.949915] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3313.950128] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3313.950327] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3313.950594] ? up_read (kernel/locking/rwsem.c:1622) [ 3313.950771] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3313.951011] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3313.951274] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3313.951530] __tcf_action_put (net/sched/act_api.c:377) [ 3313.951739] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3313.951956] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3313.952163] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3313.952445] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3313.952701] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3313.952949] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3313.953161] tcf_chain_flush (net/sched/cls_api.c:737) [ 3313.953368] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3313.953574] tcf_block_put (net/sched/cls_api.c:1543) [ 3313.953772] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3313.954014] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3313.954222] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3313.954418] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3313.954707] default_device_exit_batch (net/core/dev.c:11592) [ 3313.954963] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3313.955159] ? process_one_work (kernel/workqueue.c:2606) [ 3313.955386] process_one_work (kernel/workqueue.c:2633) [ 3313.955601] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3313.955805] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3313.956038] kthread (kernel/kthread.c:388) [ 3313.956212] ? __pfx_kthread (kernel/kthread.c:341) [ 3313.956415] ret_from_fork (arch/x86/kernel/process.c:147) [ 3313.956612] ? __pfx_kthread (kernel/kthread.c:341) [ 3313.956814] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3388.480624] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 3377481 jiffies s: 4157 root: 0x8/. | [ 3388.481252] rcu: blocking rcu_node structures (internal RCU debug): | [ 3388.481582] Sending NMI from CPU 2 to CPUs 3: | [ 3388.481838] NMI backtrace for cpu 3 [ 3388.481846] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3388.481849] Workqueue: netns cleanup_net [ 3388.481856] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:13 ./arch/x86/include/asm/vdso/processor.h:18 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3388.481862] Code: 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc 90 eb e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba All code ======== 0: 55 push %rbp 1: 41 54 push %r12 3: 55 push %rbp 4: 48 89 fd mov %rdi,%rbp 7: 53 push %rbx 8: 66 90 xchg %ax,%ax a: ba 01 00 00 00 mov $0x1,%edx f: 8b 45 00 mov 0x0(%rbp),%eax 12: 85 c0 test %eax,%eax 14: 75 14 jne 0x2a 16: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 1b: 85 c0 test %eax,%eax 1d: 75 f0 jne 0xf 1f: 5b pop %rbx 20: 5d pop %rbp 21: 41 5c pop %r12 23: 41 5d pop %r13 25: c3 ret 26: cc int3 27: cc int3 28: cc int3 29: cc int3 2a:* f3 90 pause <-- trapping instruction 2c: eb e1 jmp 0xf 2e: 81 fe 00 01 00 00 cmp $0x100,%esi 34: 74 52 je 0x88 36: 40 30 f6 xor %sil,%sil 39: 85 f6 test %esi,%esi 3b: 75 75 jne 0xb2 3d: f0 lock 3e: 0f .byte 0xf 3f: ba .byte 0xba Code starting with the faulting instruction =========================================== 0: f3 90 pause 2: eb e1 jmp 0xffffffffffffffe5 4: 81 fe 00 01 00 00 cmp $0x100,%esi a: 74 52 je 0x5e c: 40 30 f6 xor %sil,%sil f: 85 f6 test %esi,%esi 11: 75 75 jne 0x88 13: f0 lock 14: 0f .byte 0xf 15: ba .byte 0xba [ 3388.481864] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3388.481865] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3388.481866] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3388.481867] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3388.481868] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3388.481869] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3388.481872] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3388.481874] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3388.481875] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3388.481875] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3388.481876] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3388.481877] PKRU: 55555554 [ 3388.481877] Call Trace: [ 3388.481880] [ 3388.481883] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 3388.481888] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 3388.481892] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 3388.481895] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 3388.481899] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 3388.481901] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 3388.481906] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:13 ./arch/x86/include/asm/vdso/processor.h:18 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3388.481908] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:13 ./arch/x86/include/asm/vdso/processor.h:18 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3388.481910] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:13 ./arch/x86/include/asm/vdso/processor.h:18 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3388.481912] [ 3388.481913] [ 3388.481914] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3388.481919] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3388.481921] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3388.481927] ? up_read (kernel/locking/rwsem.c:1622) [ 3388.481930] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3388.481934] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3388.481936] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3388.481939] __tcf_action_put (net/sched/act_api.c:377) [ 3388.481943] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3388.481945] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3388.481947] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3388.481954] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3388.481960] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3388.481965] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3388.481967] tcf_chain_flush (net/sched/cls_api.c:737) [ 3388.481969] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3388.481971] tcf_block_put (net/sched/cls_api.c:1543) [ 3388.481974] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3388.481978] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3388.481983] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3388.481986] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3388.481990] default_device_exit_batch (net/core/dev.c:11592) [ 3388.481993] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3388.481996] ? process_one_work (kernel/workqueue.c:2606) [ 3388.482001] process_one_work (kernel/workqueue.c:2633) [ 3388.482005] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3388.482007] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3388.482009] kthread (kernel/kthread.c:388) [ 3388.482012] ? __pfx_kthread (kernel/kthread.c:341) [ 3388.482013] ret_from_fork (arch/x86/kernel/process.c:147) [ 3388.482017] ? __pfx_kthread (kernel/kthread.c:341) [ 3388.482018] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3388.482023] | [ 3391.960608] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3391.960935] rcu: 3-....: (3379684 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=774285 | [ 3391.961451] rcu: (t=3380991 jiffies g=7477 q=1775 ncpus=4) [ 3391.962194] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3391.962791] Workqueue: netns cleanup_net [ 3391.963007] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3391.963299] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3391.964266] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3391.964547] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3391.964921] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3391.965295] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3391.965671] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3391.966047] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3391.966424] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3391.966847] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3391.967152] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3391.967528] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3391.967903] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3391.968294] PKRU: 55555554 [ 3391.968443] Call Trace: [ 3391.968580] [ 3391.968695] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3391.968941] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3391.969189] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 3391.969400] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 3391.969607] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 3391.969819] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3391.970059] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3391.970295] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3391.970577] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3391.970796] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3391.971052] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3391.971293] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3391.971523] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3391.971802] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3391.972066] [ 3391.972185] [ 3391.972303] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3391.972585] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3391.972848] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3391.973055] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3391.973253] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3391.973519] ? up_read (kernel/locking/rwsem.c:1622) [ 3391.973698] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3391.973939] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3391.974203] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3391.974459] __tcf_action_put (net/sched/act_api.c:377) [ 3391.974668] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3391.974882] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3391.975088] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3391.975373] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3391.975628] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3391.975874] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3391.976087] tcf_chain_flush (net/sched/cls_api.c:737) [ 3391.976295] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3391.976501] tcf_block_put (net/sched/cls_api.c:1543) [ 3391.976696] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3391.976937] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3391.977145] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3391.977340] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3391.977627] default_device_exit_batch (net/core/dev.c:11592) [ 3391.977883] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3391.978078] ? process_one_work (kernel/workqueue.c:2606) [ 3391.978302] process_one_work (kernel/workqueue.c:2633) [ 3391.978520] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3391.978724] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3391.978954] kthread (kernel/kthread.c:388) [ 3391.979127] ? __pfx_kthread (kernel/kthread.c:341) [ 3391.979328] ret_from_fork (arch/x86/kernel/process.c:147) [ 3391.979523] ? __pfx_kthread (kernel/kthread.c:341) [ 3391.979730] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3391.979945] | [ 3469.983623] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3469.983963] rcu: 3-....: (3457689 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=792219 | [ 3469.984481] rcu: (t=3459014 jiffies g=7477 q=1775 ncpus=4) [ 3469.985233] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3469.985835] Workqueue: netns cleanup_net [ 3469.986056] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 3469.986350] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 <85> c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: f3 0f 1e fa endbr64 12: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 17: 41 55 push %r13 19: 41 54 push %r12 1b: 55 push %rbp 1c: 48 89 fd mov %rdi,%rbp 1f: 53 push %rbx 20: 66 90 xchg %ax,%ax 22: ba 01 00 00 00 mov $0x1,%edx 27: 8b 45 00 mov 0x0(%rbp),%eax 2a:* 85 c0 test %eax,%eax <-- trapping instruction 2c: 75 14 jne 0x42 2e: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 33: 85 c0 test %eax,%eax 35: 75 f0 jne 0x27 37: 5b pop %rbx 38: 5d pop %rbp 39: 41 5c pop %r12 3b: 41 5d pop %r13 3d: c3 ret 3e: cc int3 3f: cc int3 Code starting with the faulting instruction =========================================== 0: 85 c0 test %eax,%eax 2: 75 14 jne 0x18 4: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 9: 85 c0 test %eax,%eax b: 75 f0 jne 0xfffffffffffffffd d: 5b pop %rbx e: 5d pop %rbp f: 41 5c pop %r12 11: 41 5d pop %r13 13: c3 ret 14: cc int3 15: cc int3 [ 3469.987322] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3469.987603] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3469.987997] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3469.988374] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3469.988751] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3469.989125] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3469.989507] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3469.989935] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3469.990239] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3469.990617] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3469.990994] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3469.991369] PKRU: 55555554 [ 3469.991517] Call Trace: [ 3469.991659] [ 3469.991779] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3469.992018] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3469.992262] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 3469.992475] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 3469.992685] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 3469.992892] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3469.993131] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3469.993367] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3469.993654] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3469.993872] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3469.994128] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3469.994370] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3469.994599] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3469.994885] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3469.995152] [ 3469.995271] [ 3469.995388] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3469.995676] ? queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 3469.995938] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3469.996147] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3469.996346] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3469.996614] ? up_read (kernel/locking/rwsem.c:1622) [ 3469.996795] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3469.997036] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3469.997301] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3469.997558] __tcf_action_put (net/sched/act_api.c:377) [ 3469.997770] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3469.997986] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3469.998194] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3469.998475] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3469.998732] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3469.998980] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3469.999193] tcf_chain_flush (net/sched/cls_api.c:737) [ 3469.999405] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3469.999615] tcf_block_put (net/sched/cls_api.c:1543) [ 3469.999810] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3470.000051] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3470.000263] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3470.000460] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3470.000752] default_device_exit_batch (net/core/dev.c:11592) [ 3470.001013] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3470.001209] ? process_one_work (kernel/workqueue.c:2606) [ 3470.001433] process_one_work (kernel/workqueue.c:2633) [ 3470.001653] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3470.001857] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3470.002085] kthread (kernel/kthread.c:388) [ 3470.002260] ? __pfx_kthread (kernel/kthread.c:341) [ 3470.002463] ret_from_fork (arch/x86/kernel/process.c:147) [ 3470.002660] ? __pfx_kthread (kernel/kthread.c:341) [ 3470.002861] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3486.784633] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 3475785 jiffies s: 4157 root: 0x8/. | [ 3486.785499] rcu: blocking rcu_node structures (internal RCU debug): | [ 3486.785962] Sending NMI from CPU 2 to CPUs 3: | [ 3486.786302] NMI backtrace for cpu 3 [ 3486.786310] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3486.786311] Workqueue: netns cleanup_net [ 3486.786318] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3486.786322] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3486.786324] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3486.786326] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3486.786327] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3486.786328] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3486.786329] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3486.786330] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3486.786333] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3486.786334] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3486.786335] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3486.786336] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3486.786337] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3486.786337] PKRU: 55555554 [ 3486.786338] Call Trace: [ 3486.786340] [ 3486.786342] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 3486.786347] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 3486.786351] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 3486.786354] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 3486.786358] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 3486.786360] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 3486.786365] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3486.786367] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3486.786370] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3486.786372] [ 3486.786372] [ 3486.786373] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3486.786377] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3486.786378] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3486.786382] ? up_read (kernel/locking/rwsem.c:1622) [ 3486.786384] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3486.786388] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3486.786390] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3486.786392] __tcf_action_put (net/sched/act_api.c:377) [ 3486.786395] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3486.786397] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3486.786400] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3486.786405] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3486.786411] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3486.786416] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3486.786419] tcf_chain_flush (net/sched/cls_api.c:737) [ 3486.786420] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3486.786422] tcf_block_put (net/sched/cls_api.c:1543) [ 3486.786424] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3486.786429] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3486.786432] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3486.786435] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3486.786438] default_device_exit_batch (net/core/dev.c:11592) [ 3486.786441] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3486.786444] ? process_one_work (kernel/workqueue.c:2606) [ 3486.786448] process_one_work (kernel/workqueue.c:2633) [ 3486.786451] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3486.786453] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3486.786455] kthread (kernel/kthread.c:388) [ 3486.786458] ? __pfx_kthread (kernel/kthread.c:341) [ 3486.786459] ret_from_fork (arch/x86/kernel/process.c:147) [ 3486.786463] ? __pfx_kthread (kernel/kthread.c:341) [ 3486.786464] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3486.786469] | [ 3548.006609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3548.006956] rcu: 3-....: (3535679 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=810301 | [ 3548.007474] rcu: (t=3537037 jiffies g=7477 q=1775 ncpus=4) [ 3548.008227] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3548.008824] Workqueue: netns cleanup_net [ 3548.009052] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3548.009347] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3548.010310] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3548.010588] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3548.010964] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3548.011338] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3548.011714] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3548.012091] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3548.012466] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3548.012892] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3548.013195] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3548.013570] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3548.013946] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3548.014320] PKRU: 55555554 [ 3548.014467] Call Trace: [ 3548.014604] [ 3548.014722] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3548.014963] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3548.015206] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 3548.015421] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 3548.015631] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 3548.015838] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3548.016078] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3548.016312] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3548.016598] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3548.016817] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3548.017074] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3548.017316] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3548.017545] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3548.017826] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3548.018092] [ 3548.018208] [ 3548.018326] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3548.018613] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3548.018875] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3548.019087] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3548.019286] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3548.019556] ? up_read (kernel/locking/rwsem.c:1622) [ 3548.019738] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3548.019977] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3548.020242] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3548.020497] __tcf_action_put (net/sched/act_api.c:377) [ 3548.020711] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3548.020927] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3548.021134] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3548.021415] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3548.021672] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3548.021922] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3548.022135] tcf_chain_flush (net/sched/cls_api.c:737) [ 3548.022340] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3548.022550] tcf_block_put (net/sched/cls_api.c:1543) [ 3548.022745] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3548.022985] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3548.023195] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3548.023390] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3548.023682] default_device_exit_batch (net/core/dev.c:11592) [ 3548.023940] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3548.024136] ? process_one_work (kernel/workqueue.c:2606) [ 3548.024361] process_one_work (kernel/workqueue.c:2633) [ 3548.024577] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3548.024784] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3548.025013] kthread (kernel/kthread.c:388) [ 3548.025187] ? __pfx_kthread (kernel/kthread.c:341) [ 3548.025390] ret_from_fork (arch/x86/kernel/process.c:147) [ 3548.025587] ? __pfx_kthread (kernel/kthread.c:341) [ 3548.025794] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3585.088623] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 3574089 jiffies s: 4157 root: 0x8/. | [ 3585.089252] rcu: blocking rcu_node structures (internal RCU debug): | [ 3585.089603] Sending NMI from CPU 2 to CPUs 3: | [ 3585.089881] NMI backtrace for cpu 3 [ 3585.089890] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3585.089892] Workqueue: netns cleanup_net [ 3585.089900] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3585.089905] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3585.089907] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3585.089908] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3585.089910] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3585.089910] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3585.089912] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3585.089912] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3585.089916] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3585.089917] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3585.089918] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3585.089918] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3585.089919] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3585.089920] PKRU: 55555554 [ 3585.089920] Call Trace: [ 3585.089923] [ 3585.089926] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 3585.089931] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 3585.089935] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 3585.089938] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 3585.089942] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 3585.089944] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 3585.089949] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3585.089952] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3585.089954] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3585.089957] [ 3585.089957] [ 3585.089958] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3585.089964] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3585.089966] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3585.089971] ? up_read (kernel/locking/rwsem.c:1622) [ 3585.089973] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3585.089977] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3585.089979] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3585.089981] __tcf_action_put (net/sched/act_api.c:377) [ 3585.089986] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3585.089988] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3585.089990] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3585.089996] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3585.090002] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3585.090007] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3585.090010] tcf_chain_flush (net/sched/cls_api.c:737) [ 3585.090011] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3585.090013] tcf_block_put (net/sched/cls_api.c:1543) [ 3585.090015] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3585.090019] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3585.090024] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3585.090027] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3585.090030] default_device_exit_batch (net/core/dev.c:11592) [ 3585.090034] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3585.090036] ? process_one_work (kernel/workqueue.c:2606) [ 3585.090040] process_one_work (kernel/workqueue.c:2633) [ 3585.090043] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3585.090045] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3585.090047] kthread (kernel/kthread.c:388) [ 3585.090049] ? __pfx_kthread (kernel/kthread.c:341) [ 3585.090051] ret_from_fork (arch/x86/kernel/process.c:147) [ 3585.090055] ? __pfx_kthread (kernel/kthread.c:341) [ 3585.090056] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3585.090060] | [ 3626.029609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3626.029942] rcu: 3-....: (3613669 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=827926 | [ 3626.030459] rcu: (t=3615060 jiffies g=7477 q=1775 ncpus=4) [ 3626.031207] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3626.031804] Workqueue: netns cleanup_net [ 3626.032027] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3626.032320] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3626.033283] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3626.033559] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3626.033936] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3626.034309] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3626.034685] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3626.035059] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3626.035437] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3626.035861] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3626.036165] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3626.036541] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3626.036915] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3626.037288] PKRU: 55555554 [ 3626.037438] Call Trace: [ 3626.037576] [ 3626.037701] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3626.037943] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3626.038185] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 3626.038400] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 3626.038614] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 3626.038821] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3626.039061] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3626.039294] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3626.039578] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3626.039798] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3626.040053] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3626.040293] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3626.040523] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3626.040801] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3626.041066] [ 3626.041183] [ 3626.041301] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3626.041586] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3626.041848] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3626.042056] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3626.042255] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3626.042523] ? up_read (kernel/locking/rwsem.c:1622) [ 3626.042703] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3626.042942] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3626.043206] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3626.043462] __tcf_action_put (net/sched/act_api.c:377) [ 3626.043677] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3626.043894] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3626.044101] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3626.044382] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3626.044639] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3626.044886] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3626.045098] tcf_chain_flush (net/sched/cls_api.c:737) [ 3626.045303] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3626.045513] tcf_block_put (net/sched/cls_api.c:1543) [ 3626.045709] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3626.045949] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3626.046159] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3626.046354] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3626.046643] default_device_exit_batch (net/core/dev.c:11592) [ 3626.046900] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3626.047095] ? process_one_work (kernel/workqueue.c:2606) [ 3626.047320] process_one_work (kernel/workqueue.c:2633) [ 3626.047536] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3626.047742] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3626.047971] kthread (kernel/kthread.c:388) [ 3626.048145] ? __pfx_kthread (kernel/kthread.c:341) [ 3626.048347] ret_from_fork (arch/x86/kernel/process.c:147) [ 3626.048549] ? __pfx_kthread (kernel/kthread.c:341) [ 3626.048757] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3683.392620] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 3672393 jiffies s: 4157 root: 0x8/. | [ 3683.393228] rcu: blocking rcu_node structures (internal RCU debug): | [ 3683.393560] Sending NMI from CPU 2 to CPUs 3: | [ 3683.393811] NMI backtrace for cpu 3 [ 3683.393818] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3683.393821] Workqueue: netns cleanup_net [ 3683.393829] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3683.393835] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3683.393837] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3683.393838] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3683.393839] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3683.393841] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3683.393842] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3683.393842] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3683.393846] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3683.393847] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3683.393848] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3683.393849] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3683.393849] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3683.393850] PKRU: 55555554 [ 3683.393851] Call Trace: [ 3683.393853] [ 3683.393856] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 3683.393861] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 3683.393865] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 3683.393869] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 3683.393873] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 3683.393876] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 3683.393881] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3683.393884] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3683.393886] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3683.393888] [ 3683.393889] [ 3683.393890] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3683.393895] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3683.393897] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3683.393902] ? up_read (kernel/locking/rwsem.c:1622) [ 3683.393905] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3683.393909] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3683.393911] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3683.393913] __tcf_action_put (net/sched/act_api.c:377) [ 3683.393918] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3683.393920] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3683.393922] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3683.393928] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3683.393934] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3683.393939] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3683.393942] tcf_chain_flush (net/sched/cls_api.c:737) [ 3683.393944] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3683.393946] tcf_block_put (net/sched/cls_api.c:1543) [ 3683.393948] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3683.393953] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3683.393958] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3683.393961] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3683.393965] default_device_exit_batch (net/core/dev.c:11592) [ 3683.393968] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3683.393970] ? process_one_work (kernel/workqueue.c:2606) [ 3683.393974] process_one_work (kernel/workqueue.c:2633) [ 3683.393977] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3683.393979] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3683.393981] kthread (kernel/kthread.c:388) [ 3683.393984] ? __pfx_kthread (kernel/kthread.c:341) [ 3683.393985] ret_from_fork (arch/x86/kernel/process.c:147) [ 3683.393988] ? __pfx_kthread (kernel/kthread.c:341) [ 3683.393990] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3683.393996] | [ 3704.051609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3704.051951] rcu: 3-....: (3691658 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=847094 | [ 3704.052471] rcu: (t=3693082 jiffies g=7477 q=1775 ncpus=4) [ 3704.053219] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3704.053818] Workqueue: netns cleanup_net [ 3704.054037] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3704.054332] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3704.055300] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3704.055575] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3704.055950] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3704.056325] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3704.056697] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3704.057073] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3704.057450] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3704.057872] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3704.058181] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3704.058558] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3704.058934] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3704.059308] PKRU: 55555554 [ 3704.059457] Call Trace: [ 3704.059594] [ 3704.059714] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3704.059953] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3704.060197] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 3704.060410] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 3704.060626] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 3704.060834] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3704.061071] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3704.061309] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3704.061591] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3704.061814] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3704.062070] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3704.062309] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3704.062542] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3704.062821] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3704.063087] [ 3704.063205] [ 3704.063325] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3704.063611] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3704.063885] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3704.064094] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3704.064294] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3704.064561] ? up_read (kernel/locking/rwsem.c:1622) [ 3704.064741] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3704.064984] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3704.065248] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3704.065505] __tcf_action_put (net/sched/act_api.c:377) [ 3704.065716] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3704.065932] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3704.066142] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3704.066424] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3704.066682] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3704.066929] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3704.067142] tcf_chain_flush (net/sched/cls_api.c:737) [ 3704.067349] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3704.067556] tcf_block_put (net/sched/cls_api.c:1543) [ 3704.067753] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3704.068000] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3704.068216] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3704.068412] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3704.068703] default_device_exit_batch (net/core/dev.c:11592) [ 3704.068959] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3704.069158] ? process_one_work (kernel/workqueue.c:2606) [ 3704.069387] process_one_work (kernel/workqueue.c:2633) [ 3704.069603] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3704.069811] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3704.070043] kthread (kernel/kthread.c:388) [ 3704.070217] ? __pfx_kthread (kernel/kthread.c:341) [ 3704.070421] ret_from_fork (arch/x86/kernel/process.c:147) [ 3704.070619] ? __pfx_kthread (kernel/kthread.c:341) [ 3704.070822] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3781.696619] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 3770697 jiffies s: 4157 root: 0x8/. | [ 3781.697239] rcu: blocking rcu_node structures (internal RCU debug): | [ 3781.697581] Sending NMI from CPU 2 to CPUs 3: | [ 3781.697842] NMI backtrace for cpu 3 [ 3781.697851] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3781.697853] Workqueue: netns cleanup_net [ 3781.697861] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3781.697868] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3781.697870] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3781.697871] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3781.697873] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3781.697874] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3781.697875] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3781.697876] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3781.697879] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3781.697880] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3781.697881] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3781.697882] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3781.697882] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3781.697883] PKRU: 55555554 [ 3781.697884] Call Trace: [ 3781.697887] [ 3781.697889] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 3781.697894] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 3781.697899] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 3781.697902] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 3781.697906] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 3781.697908] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 3781.697913] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3781.697916] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3781.697918] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3781.697921] [ 3781.697921] [ 3781.697922] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3781.697928] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3781.697929] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3781.697935] ? up_read (kernel/locking/rwsem.c:1622) [ 3781.697938] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3781.697943] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3781.697944] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3781.697946] __tcf_action_put (net/sched/act_api.c:377) [ 3781.697951] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3781.697954] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3781.697956] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3781.697962] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3781.697968] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3781.697973] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3781.697976] tcf_chain_flush (net/sched/cls_api.c:737) [ 3781.697978] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3781.697980] tcf_block_put (net/sched/cls_api.c:1543) [ 3781.697982] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3781.697987] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3781.697992] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3781.697995] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3781.697999] default_device_exit_batch (net/core/dev.c:11592) [ 3781.698002] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3781.698005] ? process_one_work (kernel/workqueue.c:2606) [ 3781.698009] process_one_work (kernel/workqueue.c:2633) [ 3781.698012] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3781.698014] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3781.698016] kthread (kernel/kthread.c:388) [ 3781.698019] ? __pfx_kthread (kernel/kthread.c:341) [ 3781.698021] ret_from_fork (arch/x86/kernel/process.c:147) [ 3781.698025] ? __pfx_kthread (kernel/kthread.c:341) [ 3781.698026] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3781.698031] | [ 3782.074609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3782.074930] rcu: 3-....: (3769648 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=864686 | [ 3782.075449] rcu: (t=3771105 jiffies g=7477 q=1775 ncpus=4) [ 3782.076194] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3782.076788] Workqueue: netns cleanup_net [ 3782.077004] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3782.077297] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3782.078266] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3782.078544] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3782.078921] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3782.079295] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3782.079672] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3782.080046] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3782.080421] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3782.080850] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3782.081156] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3782.081531] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3782.081907] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3782.082282] PKRU: 55555554 [ 3782.082429] Call Trace: [ 3782.082566] [ 3782.082683] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3782.082925] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3782.083171] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 3782.083383] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 3782.083591] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 3782.083799] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3782.084040] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3782.084277] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3782.084561] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3782.084781] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3782.085037] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3782.085279] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3782.085511] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3782.085792] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3782.086057] [ 3782.086176] [ 3782.086295] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3782.086578] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3782.086841] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3782.087049] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3782.087247] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3782.087514] ? up_read (kernel/locking/rwsem.c:1622) [ 3782.087698] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3782.087946] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3782.088211] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3782.088471] __tcf_action_put (net/sched/act_api.c:377) [ 3782.088681] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3782.088896] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3782.089102] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3782.089386] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3782.089642] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3782.089889] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3782.090102] tcf_chain_flush (net/sched/cls_api.c:737) [ 3782.090310] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3782.090521] tcf_block_put (net/sched/cls_api.c:1543) [ 3782.090725] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3782.090975] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3782.091195] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3782.091398] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3782.091697] default_device_exit_batch (net/core/dev.c:11592) [ 3782.091963] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3782.092159] ? process_one_work (kernel/workqueue.c:2606) [ 3782.092384] process_one_work (kernel/workqueue.c:2633) [ 3782.092603] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3782.092808] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3782.093038] kthread (kernel/kthread.c:388) [ 3782.093211] ? __pfx_kthread (kernel/kthread.c:341) [ 3782.093414] ret_from_fork (arch/x86/kernel/process.c:147) [ 3782.093609] ? __pfx_kthread (kernel/kthread.c:341) [ 3782.093812] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3782.094029] | [ 3860.097609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3860.097954] rcu: 3-....: (3847653 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=882227 | [ 3860.098473] rcu: (t=3849128 jiffies g=7477 q=1775 ncpus=4) [ 3860.099220] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3860.099816] Workqueue: netns cleanup_net [ 3860.100033] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 3860.100327] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 <85> c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: f3 0f 1e fa endbr64 12: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 17: 41 55 push %r13 19: 41 54 push %r12 1b: 55 push %rbp 1c: 48 89 fd mov %rdi,%rbp 1f: 53 push %rbx 20: 66 90 xchg %ax,%ax 22: ba 01 00 00 00 mov $0x1,%edx 27: 8b 45 00 mov 0x0(%rbp),%eax 2a:* 85 c0 test %eax,%eax <-- trapping instruction 2c: 75 14 jne 0x42 2e: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 33: 85 c0 test %eax,%eax 35: 75 f0 jne 0x27 37: 5b pop %rbx 38: 5d pop %rbp 39: 41 5c pop %r12 3b: 41 5d pop %r13 3d: c3 ret 3e: cc int3 3f: cc int3 Code starting with the faulting instruction =========================================== 0: 85 c0 test %eax,%eax 2: 75 14 jne 0x18 4: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 9: 85 c0 test %eax,%eax b: 75 f0 jne 0xfffffffffffffffd d: 5b pop %rbx e: 5d pop %rbp f: 41 5c pop %r12 11: 41 5d pop %r13 13: c3 ret 14: cc int3 15: cc int3 [ 3860.101291] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3860.101567] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3860.101945] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3860.102320] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3860.102694] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3860.103067] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3860.103445] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3860.103869] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3860.104177] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3860.104570] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3860.104963] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3860.105356] PKRU: 55555554 [ 3860.105510] Call Trace: [ 3860.105652] [ 3860.105774] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3860.106021] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3860.106265] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 3860.106479] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 3860.106689] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 3860.106896] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3860.107137] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3860.107380] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3860.107666] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3860.107886] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3860.108143] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3860.108382] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3860.108614] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3860.108892] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3860.109158] [ 3860.109277] [ 3860.109396] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3860.109680] ? queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 3860.109942] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3860.110152] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3860.110351] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3860.110620] ? up_read (kernel/locking/rwsem.c:1622) [ 3860.110798] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3860.111039] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3860.111304] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3860.111560] __tcf_action_put (net/sched/act_api.c:377) [ 3860.111770] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3860.111988] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3860.112196] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3860.112478] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3860.112735] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3860.112982] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3860.113194] tcf_chain_flush (net/sched/cls_api.c:737) [ 3860.113401] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3860.113607] tcf_block_put (net/sched/cls_api.c:1543) [ 3860.113803] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3860.114047] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3860.114256] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3860.114451] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3860.114741] default_device_exit_batch (net/core/dev.c:11592) [ 3860.114998] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3860.115195] ? process_one_work (kernel/workqueue.c:2606) [ 3860.115421] process_one_work (kernel/workqueue.c:2633) [ 3860.115638] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3860.115842] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3860.116074] kthread (kernel/kthread.c:388) [ 3860.116249] ? __pfx_kthread (kernel/kthread.c:341) [ 3860.116452] ret_from_fork (arch/x86/kernel/process.c:147) [ 3860.116649] ? __pfx_kthread (kernel/kthread.c:341) [ 3860.116852] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3880.000619] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 3869001 jiffies s: 4157 root: 0x8/. | [ 3880.001228] rcu: blocking rcu_node structures (internal RCU debug): | [ 3880.001556] Sending NMI from CPU 2 to CPUs 3: | [ 3880.001806] NMI backtrace for cpu 3 [ 3880.001813] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3880.001815] Workqueue: netns cleanup_net [ 3880.001821] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3880.001826] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3880.001828] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3880.001829] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3880.001830] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3880.001831] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3880.001832] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3880.001833] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3880.001836] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3880.001837] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3880.001838] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3880.001839] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3880.001839] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3880.001840] PKRU: 55555554 [ 3880.001841] Call Trace: [ 3880.001843] [ 3880.001845] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 3880.001850] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 3880.001854] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 3880.001857] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 3880.001861] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 3880.001863] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 3880.001867] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3880.001870] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3880.001872] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3880.001875] [ 3880.001875] [ 3880.001876] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3880.001880] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3880.001882] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3880.001885] ? up_read (kernel/locking/rwsem.c:1622) [ 3880.001888] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3880.001892] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3880.001894] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3880.001897] __tcf_action_put (net/sched/act_api.c:377) [ 3880.001900] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3880.001902] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3880.001904] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3880.001910] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3880.001916] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3880.001920] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3880.001923] tcf_chain_flush (net/sched/cls_api.c:737) [ 3880.001924] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3880.001926] tcf_block_put (net/sched/cls_api.c:1543) [ 3880.001928] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3880.001932] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3880.001935] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3880.001937] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3880.001941] default_device_exit_batch (net/core/dev.c:11592) [ 3880.001943] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3880.001946] ? process_one_work (kernel/workqueue.c:2606) [ 3880.001949] process_one_work (kernel/workqueue.c:2633) [ 3880.001952] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3880.001954] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3880.001956] kthread (kernel/kthread.c:388) [ 3880.001958] ? __pfx_kthread (kernel/kthread.c:341) [ 3880.001960] ret_from_fork (arch/x86/kernel/process.c:147) [ 3880.001963] ? __pfx_kthread (kernel/kthread.c:341) [ 3880.001964] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3880.001969] | [ 3938.120621] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3938.120964] rcu: 3-....: (3925643 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=899947 | [ 3938.121483] rcu: (t=3927151 jiffies g=7477 q=1776 ncpus=4) [ 3938.122236] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3938.122834] Workqueue: netns cleanup_net [ 3938.123054] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3938.123351] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3938.124315] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3938.124593] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3938.124966] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3938.125341] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3938.125714] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3938.126086] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3938.126459] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3938.126886] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3938.127195] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3938.127574] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3938.127952] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3938.128327] PKRU: 55555554 [ 3938.128475] Call Trace: [ 3938.128618] [ 3938.128739] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3938.128981] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3938.129226] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 3938.129438] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 3938.129648] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 3938.129854] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3938.130094] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3938.130332] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3938.130616] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3938.130835] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3938.131093] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3938.131334] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3938.131567] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3938.131847] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3938.132112] [ 3938.132231] [ 3938.132350] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3938.132634] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3938.132896] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3938.133104] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3938.133303] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3938.133571] ? up_read (kernel/locking/rwsem.c:1622) [ 3938.133751] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3938.133995] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3938.134259] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3938.134516] __tcf_action_put (net/sched/act_api.c:377) [ 3938.134726] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3938.134941] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3938.135149] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3938.135433] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3938.135691] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3938.135937] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3938.136151] tcf_chain_flush (net/sched/cls_api.c:737) [ 3938.136359] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3938.136566] tcf_block_put (net/sched/cls_api.c:1543) [ 3938.136762] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3938.137008] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3938.137218] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3938.137413] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3938.137703] default_device_exit_batch (net/core/dev.c:11592) [ 3938.137961] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3938.138156] ? process_one_work (kernel/workqueue.c:2606) [ 3938.138383] process_one_work (kernel/workqueue.c:2633) [ 3938.138611] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3938.138814] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3938.139047] kthread (kernel/kthread.c:388) [ 3938.139220] ? __pfx_kthread (kernel/kthread.c:341) [ 3938.139424] ret_from_fork (arch/x86/kernel/process.c:147) [ 3938.139622] ? __pfx_kthread (kernel/kthread.c:341) [ 3938.139831] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3978.304628] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 3967305 jiffies s: 4157 root: 0x8/. | [ 3978.305240] rcu: blocking rcu_node structures (internal RCU debug): | [ 3978.305575] Sending NMI from CPU 2 to CPUs 3: | [ 3978.305837] NMI backtrace for cpu 3 [ 3978.305845] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3978.305847] Workqueue: netns cleanup_net [ 3978.305854] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3978.305860] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3978.305861] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 3978.305862] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 3978.305863] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 3978.305865] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 3978.305866] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 3978.305866] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 3978.305869] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 3978.305870] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3978.305871] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 3978.305872] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3978.305873] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3978.305873] PKRU: 55555554 [ 3978.305874] Call Trace: [ 3978.305876] [ 3978.305878] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 3978.305883] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 3978.305887] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 3978.305890] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 3978.305894] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 3978.305896] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 3978.305901] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3978.305903] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3978.305906] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3978.305908] [ 3978.305908] [ 3978.305909] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3978.305914] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3978.305916] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3978.305920] ? up_read (kernel/locking/rwsem.c:1622) [ 3978.305924] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3978.305928] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3978.305930] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3978.305932] __tcf_action_put (net/sched/act_api.c:377) [ 3978.305936] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3978.305938] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3978.305941] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3978.305947] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3978.305954] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3978.305959] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3978.305962] tcf_chain_flush (net/sched/cls_api.c:737) [ 3978.305963] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3978.305966] tcf_block_put (net/sched/cls_api.c:1543) [ 3978.305968] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3978.305973] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3978.305978] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3978.305981] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 3978.305985] default_device_exit_batch (net/core/dev.c:11592) [ 3978.305988] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3978.305990] ? process_one_work (kernel/workqueue.c:2606) [ 3978.305993] process_one_work (kernel/workqueue.c:2633) [ 3978.305997] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3978.305999] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3978.306001] kthread (kernel/kthread.c:388) [ 3978.306003] ? __pfx_kthread (kernel/kthread.c:341) [ 3978.306004] ret_from_fork (arch/x86/kernel/process.c:147) [ 3978.306008] ? __pfx_kthread (kernel/kthread.c:341) [ 3978.306009] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3978.306013] | [ 4016.142610] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4016.142957] rcu: 3-....: (4003632 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=918074 | [ 4016.143474] rcu: (t=4005173 jiffies g=7477 q=1776 ncpus=4) [ 4016.144222] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4016.144817] Workqueue: netns cleanup_net [ 4016.145050] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4016.145347] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4016.146345] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 4016.146627] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 4016.147001] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 4016.147375] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 4016.147747] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 4016.148122] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 4016.148498] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 4016.148921] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4016.149228] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 4016.149603] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4016.149979] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4016.150351] PKRU: 55555554 [ 4016.150500] Call Trace: [ 4016.150638] [ 4016.150755] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4016.150993] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4016.151237] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 4016.151449] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 4016.151660] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 4016.151867] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4016.152104] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4016.152340] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4016.152623] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4016.152844] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4016.153100] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4016.153338] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4016.153571] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4016.153849] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4016.154113] [ 4016.154231] [ 4016.154350] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4016.154633] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4016.154894] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4016.155102] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4016.155300] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4016.155569] ? up_read (kernel/locking/rwsem.c:1622) [ 4016.155747] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4016.155989] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4016.156253] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4016.156509] __tcf_action_put (net/sched/act_api.c:377) [ 4016.156725] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4016.156942] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4016.157150] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4016.157430] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4016.157688] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4016.157934] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4016.158146] tcf_chain_flush (net/sched/cls_api.c:737) [ 4016.158352] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4016.158558] tcf_block_put (net/sched/cls_api.c:1543) [ 4016.158752] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4016.158994] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4016.159202] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4016.159397] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 4016.159686] default_device_exit_batch (net/core/dev.c:11592) [ 4016.159941] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4016.160136] ? process_one_work (kernel/workqueue.c:2606) [ 4016.160365] process_one_work (kernel/workqueue.c:2633) [ 4016.160579] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4016.160783] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4016.161017] kthread (kernel/kthread.c:388) [ 4016.161190] ? __pfx_kthread (kernel/kthread.c:341) [ 4016.161393] ret_from_fork (arch/x86/kernel/process.c:147) [ 4016.161589] ? __pfx_kthread (kernel/kthread.c:341) [ 4016.161792] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4076.608619] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 4065609 jiffies s: 4157 root: 0x8/. | [ 4076.609238] rcu: blocking rcu_node structures (internal RCU debug): | [ 4076.609574] Sending NMI from CPU 2 to CPUs 3: | [ 4076.609826] NMI backtrace for cpu 3 [ 4076.609834] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4076.609836] Workqueue: netns cleanup_net [ 4076.609843] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4076.609849] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4076.609851] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 4076.609853] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 4076.609854] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 4076.609855] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 4076.609856] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 4076.609857] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 4076.609860] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 4076.609861] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4076.609862] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 4076.609862] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4076.609863] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4076.609864] PKRU: 55555554 [ 4076.609865] Call Trace: [ 4076.609867] [ 4076.609870] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 4076.609875] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 4076.609880] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 4076.609883] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 4076.609887] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 4076.609889] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 4076.609893] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4076.609895] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4076.609897] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4076.609899] [ 4076.609900] [ 4076.609901] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4076.609906] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4076.609908] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4076.609912] ? up_read (kernel/locking/rwsem.c:1622) [ 4076.609915] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4076.609919] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4076.609921] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4076.609923] __tcf_action_put (net/sched/act_api.c:377) [ 4076.609927] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4076.609929] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4076.609931] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4076.609937] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4076.609943] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4076.609949] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4076.609951] tcf_chain_flush (net/sched/cls_api.c:737) [ 4076.609953] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4076.609955] tcf_block_put (net/sched/cls_api.c:1543) [ 4076.609958] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4076.609962] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4076.609967] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4076.609970] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 4076.609974] default_device_exit_batch (net/core/dev.c:11592) [ 4076.609977] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4076.609979] ? process_one_work (kernel/workqueue.c:2606) [ 4076.609983] process_one_work (kernel/workqueue.c:2633) [ 4076.609986] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4076.609988] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4076.609990] kthread (kernel/kthread.c:388) [ 4076.609992] ? __pfx_kthread (kernel/kthread.c:341) [ 4076.609994] ret_from_fork (arch/x86/kernel/process.c:147) [ 4076.609997] ? __pfx_kthread (kernel/kthread.c:341) [ 4076.609998] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4076.610003] | [ 4094.164609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4094.164939] rcu: 3-....: (4081621 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=937026 | [ 4094.165455] rcu: (t=4083195 jiffies g=7477 q=1777 ncpus=4) [ 4094.166206] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4094.166802] Workqueue: netns cleanup_net [ 4094.167018] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4094.167310] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4094.168272] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 4094.168548] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 4094.168924] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 4094.169298] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 4094.169674] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 4094.170045] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 4094.170421] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 4094.170847] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4094.171148] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 4094.171523] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4094.171898] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4094.172272] PKRU: 55555554 [ 4094.172419] Call Trace: [ 4094.172555] [ 4094.172673] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4094.172914] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4094.173157] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 4094.173370] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 4094.173579] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 4094.173786] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4094.174025] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4094.174260] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4094.174544] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4094.174763] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4094.175020] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4094.175260] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4094.175490] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4094.175768] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4094.176033] [ 4094.176150] [ 4094.176274] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4094.176558] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4094.176820] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4094.177028] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4094.177227] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4094.177493] ? up_read (kernel/locking/rwsem.c:1622) [ 4094.177673] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4094.177913] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4094.178178] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4094.178433] __tcf_action_put (net/sched/act_api.c:377) [ 4094.178645] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4094.178861] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4094.179068] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4094.179350] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4094.179605] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4094.179855] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4094.180066] tcf_chain_flush (net/sched/cls_api.c:737) [ 4094.180272] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4094.180480] tcf_block_put (net/sched/cls_api.c:1543) [ 4094.180677] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4094.180919] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4094.181127] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4094.181323] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 4094.181612] default_device_exit_batch (net/core/dev.c:11592) [ 4094.181870] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4094.182066] ? process_one_work (kernel/workqueue.c:2606) [ 4094.182291] process_one_work (kernel/workqueue.c:2633) [ 4094.182507] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4094.182713] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4094.182944] kthread (kernel/kthread.c:388) [ 4094.183118] ? __pfx_kthread (kernel/kthread.c:341) [ 4094.183320] ret_from_fork (arch/x86/kernel/process.c:147) [ 4094.183517] ? __pfx_kthread (kernel/kthread.c:341) [ 4094.183724] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4094.183939] | [ 4172.186610] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4172.186948] rcu: 3-....: (4159625 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=956380 | [ 4172.187466] rcu: (t=4161217 jiffies g=7477 q=1777 ncpus=4) [ 4172.188222] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4172.188818] Workqueue: netns cleanup_net [ 4172.189042] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4172.189336] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4172.190304] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 4172.190581] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 4172.190957] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 4172.191332] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 4172.191708] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 4172.192083] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 4172.192459] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 4172.192883] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4172.193188] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 4172.193562] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4172.193939] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4172.194311] PKRU: 55555554 [ 4172.194462] Call Trace: [ 4172.194601] [ 4172.194725] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4172.194966] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4172.195210] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 4172.195424] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 4172.195635] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 4172.195852] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4172.196092] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4172.196326] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4172.196610] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4172.196830] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4172.197087] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4172.197327] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4172.197558] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4172.197838] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4172.198103] [ 4172.198222] [ 4172.198341] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4172.198630] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4172.198891] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4172.199100] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4172.199298] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4172.199566] ? up_read (kernel/locking/rwsem.c:1622) [ 4172.199745] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4172.199987] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4172.200251] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4172.200507] __tcf_action_put (net/sched/act_api.c:377) [ 4172.200721] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4172.200940] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4172.201147] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4172.201430] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4172.201688] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4172.201935] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4172.202147] tcf_chain_flush (net/sched/cls_api.c:737) [ 4172.202354] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4172.202562] tcf_block_put (net/sched/cls_api.c:1543) [ 4172.202759] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4172.203001] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4172.203210] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4172.203406] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 4172.203698] default_device_exit_batch (net/core/dev.c:11592) [ 4172.203956] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4172.204151] ? process_one_work (kernel/workqueue.c:2606) [ 4172.204378] process_one_work (kernel/workqueue.c:2633) [ 4172.204594] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4172.204800] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4172.205028] kthread (kernel/kthread.c:388) [ 4172.205203] ? __pfx_kthread (kernel/kthread.c:341) [ 4172.205406] ret_from_fork (arch/x86/kernel/process.c:147) [ 4172.205602] ? __pfx_kthread (kernel/kthread.c:341) [ 4172.205813] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4174.912617] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 4163913 jiffies s: 4157 root: 0x8/. | [ 4174.913229] rcu: blocking rcu_node structures (internal RCU debug): | [ 4174.913559] Sending NMI from CPU 2 to CPUs 3: | [ 4174.913811] NMI backtrace for cpu 3 [ 4174.913816] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4174.913818] Workqueue: netns cleanup_net [ 4174.913821] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4174.913824] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4174.913825] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 4174.913826] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 4174.913827] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 4174.913828] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 4174.913829] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 4174.913829] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 4174.913832] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 4174.913833] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4174.913834] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 4174.913835] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4174.913836] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4174.913837] PKRU: 55555554 [ 4174.913838] Call Trace: [ 4174.913839] [ 4174.913840] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 4174.913845] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 4174.913849] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 4174.913852] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 4174.913857] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 4174.913859] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 4174.913862] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4174.913865] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4174.913867] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4174.913869] [ 4174.913870] [ 4174.913871] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4174.913873] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4174.913874] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4174.913877] ? up_read (kernel/locking/rwsem.c:1622) [ 4174.913879] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4174.913882] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4174.913883] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4174.913885] __tcf_action_put (net/sched/act_api.c:377) [ 4174.913888] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4174.913890] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4174.913891] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4174.913896] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4174.913902] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4174.913907] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4174.913909] tcf_chain_flush (net/sched/cls_api.c:737) [ 4174.913911] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4174.913913] tcf_block_put (net/sched/cls_api.c:1543) [ 4174.913915] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4174.913918] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4174.913921] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4174.913923] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 4174.913926] default_device_exit_batch (net/core/dev.c:11592) [ 4174.913929] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4174.913931] ? process_one_work (kernel/workqueue.c:2606) [ 4174.913933] process_one_work (kernel/workqueue.c:2633) [ 4174.913936] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4174.913939] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4174.913941] kthread (kernel/kthread.c:388) [ 4174.913942] ? __pfx_kthread (kernel/kthread.c:341) [ 4174.913944] ret_from_fork (arch/x86/kernel/process.c:147) [ 4174.913947] ? __pfx_kthread (kernel/kthread.c:341) [ 4174.913948] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4174.913952] | [ 4250.208609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4250.208958] rcu: 3-....: (4237614 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=975372 | [ 4250.209476] rcu: (t=4239239 jiffies g=7477 q=1777 ncpus=4) [ 4250.210227] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4250.210825] Workqueue: netns cleanup_net [ 4250.211044] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 4250.211338] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 <8b> 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: 90 nop f: 90 nop 10: 90 nop 11: f3 0f 1e fa endbr64 15: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 1a: 41 55 push %r13 1c: 41 54 push %r12 1e: 55 push %rbp 1f: 48 89 fd mov %rdi,%rbp 22: 53 push %rbx 23: 66 90 xchg %ax,%ax 25: ba 01 00 00 00 mov $0x1,%edx 2a:* 8b 45 00 mov 0x0(%rbp),%eax <-- trapping instruction 2d: 85 c0 test %eax,%eax 2f: 75 14 jne 0x45 31: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 36: 85 c0 test %eax,%eax 38: 75 f0 jne 0x2a 3a: 5b pop %rbx 3b: 5d pop %rbp 3c: 41 5c pop %r12 3e: 41 5d pop %r13 Code starting with the faulting instruction =========================================== 0: 8b 45 00 mov 0x0(%rbp),%eax 3: 85 c0 test %eax,%eax 5: 75 14 jne 0x1b 7: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) c: 85 c0 test %eax,%eax e: 75 f0 jne 0x0 10: 5b pop %rbx 11: 5d pop %rbp 12: 41 5c pop %r12 14: 41 5d pop %r13 [ 4250.212306] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 4250.212583] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 4250.212959] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 4250.213334] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 4250.213709] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 4250.214083] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 4250.214459] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 4250.214883] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4250.215190] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 4250.215570] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4250.215949] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4250.216323] PKRU: 55555554 [ 4250.216471] Call Trace: [ 4250.216608] [ 4250.216726] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4250.216965] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4250.217209] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 4250.217420] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 4250.217631] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 4250.217838] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4250.218077] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4250.218313] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4250.218597] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4250.218818] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4250.219075] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4250.219313] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4250.219545] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4250.219824] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4250.220088] [ 4250.220207] [ 4250.220326] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4250.220609] ? queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 4250.220871] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4250.221081] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4250.221280] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4250.221549] ? up_read (kernel/locking/rwsem.c:1622) [ 4250.221726] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4250.221968] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4250.222231] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4250.222487] __tcf_action_put (net/sched/act_api.c:377) [ 4250.222697] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4250.222915] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4250.223122] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4250.223404] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4250.223661] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4250.223911] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4250.224124] tcf_chain_flush (net/sched/cls_api.c:737) [ 4250.224330] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4250.224538] tcf_block_put (net/sched/cls_api.c:1543) [ 4250.224735] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4250.224983] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4250.225202] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4250.225407] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 4250.225710] default_device_exit_batch (net/core/dev.c:11592) [ 4250.225977] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4250.226182] ? process_one_work (kernel/workqueue.c:2606) [ 4250.226419] process_one_work (kernel/workqueue.c:2633) [ 4250.226645] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4250.226848] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4250.227081] kthread (kernel/kthread.c:388) [ 4250.227256] ? __pfx_kthread (kernel/kthread.c:341) [ 4250.227459] ret_from_fork (arch/x86/kernel/process.c:147) [ 4250.227657] ? __pfx_kthread (kernel/kthread.c:341) [ 4250.227859] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4273.216618] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 4262217 jiffies s: 4157 root: 0x8/. | [ 4273.217227] rcu: blocking rcu_node structures (internal RCU debug): | [ 4273.217563] Sending NMI from CPU 2 to CPUs 3: | [ 4273.217814] NMI backtrace for cpu 3 [ 4273.217822] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4273.217824] Workqueue: netns cleanup_net [ 4273.217830] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4273.217835] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4273.217837] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 4273.217838] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 4273.217839] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 4273.217840] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 4273.217841] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 4273.217842] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 4273.217845] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 4273.217846] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4273.217847] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 4273.217848] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4273.217849] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4273.217850] PKRU: 55555554 [ 4273.217851] Call Trace: [ 4273.217853] [ 4273.217855] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 4273.217860] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 4273.217864] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 4273.217868] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 4273.217872] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 4273.217874] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 4273.217878] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4273.217880] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4273.217883] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4273.217885] [ 4273.217885] [ 4273.217887] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4273.217892] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4273.217894] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4273.217898] ? up_read (kernel/locking/rwsem.c:1622) [ 4273.217900] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4273.217904] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4273.217905] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4273.217907] __tcf_action_put (net/sched/act_api.c:377) [ 4273.217912] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4273.217914] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4273.217916] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4273.217921] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4273.217927] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4273.217932] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4273.217935] tcf_chain_flush (net/sched/cls_api.c:737) [ 4273.217937] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4273.217939] tcf_block_put (net/sched/cls_api.c:1543) [ 4273.217941] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4273.217946] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4273.217950] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4273.217952] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 4273.217956] default_device_exit_batch (net/core/dev.c:11592) [ 4273.217959] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4273.217961] ? process_one_work (kernel/workqueue.c:2606) [ 4273.217965] process_one_work (kernel/workqueue.c:2633) [ 4273.217968] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4273.217970] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4273.217972] kthread (kernel/kthread.c:388) [ 4273.217974] ? __pfx_kthread (kernel/kthread.c:341) [ 4273.217976] ret_from_fork (arch/x86/kernel/process.c:147) [ 4273.217979] ? __pfx_kthread (kernel/kthread.c:341) [ 4273.217981] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4273.217986] | [ 4328.231609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4328.231953] rcu: 3-....: (4315604 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=994763 | [ 4328.232471] rcu: (t=4317262 jiffies g=7477 q=1777 ncpus=4) [ 4328.233215] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4328.233809] Workqueue: netns cleanup_net [ 4328.234027] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4328.234324] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4328.235295] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 4328.235574] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 4328.235947] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 4328.236321] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 4328.236697] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 4328.237071] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 4328.237446] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 4328.237871] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4328.238175] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 4328.238550] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4328.238924] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4328.239299] PKRU: 55555554 [ 4328.239446] Call Trace: [ 4328.239585] [ 4328.239703] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4328.239946] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4328.240192] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 4328.240406] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 4328.240622] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 4328.240836] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4328.241087] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4328.241332] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4328.241628] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4328.241847] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4328.242102] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4328.242343] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4328.242574] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4328.242856] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4328.243121] [ 4328.243240] [ 4328.243358] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4328.243644] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4328.243907] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4328.244115] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4328.244314] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4328.244581] ? up_read (kernel/locking/rwsem.c:1622) [ 4328.244762] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4328.245007] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4328.245272] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4328.245528] __tcf_action_put (net/sched/act_api.c:377) [ 4328.245739] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4328.245954] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4328.246161] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4328.246443] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4328.246699] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4328.246946] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4328.247157] tcf_chain_flush (net/sched/cls_api.c:737) [ 4328.247365] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4328.247573] tcf_block_put (net/sched/cls_api.c:1543) [ 4328.247768] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4328.248008] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4328.248219] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4328.248414] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 4328.248704] default_device_exit_batch (net/core/dev.c:11592) [ 4328.248962] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4328.249158] ? process_one_work (kernel/workqueue.c:2606) [ 4328.249382] process_one_work (kernel/workqueue.c:2633) [ 4328.249601] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4328.249805] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4328.250033] kthread (kernel/kthread.c:388) [ 4328.250208] ? __pfx_kthread (kernel/kthread.c:341) [ 4328.250410] ret_from_fork (arch/x86/kernel/process.c:147) [ 4328.250608] ? __pfx_kthread (kernel/kthread.c:341) [ 4328.250811] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4371.520619] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 4360521 jiffies s: 4157 root: 0x8/. | [ 4371.521235] rcu: blocking rcu_node structures (internal RCU debug): | [ 4371.521568] Sending NMI from CPU 2 to CPUs 3: | [ 4371.521821] NMI backtrace for cpu 3 [ 4371.521829] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4371.521831] Workqueue: netns cleanup_net [ 4371.521838] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4371.521845] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4371.521847] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 4371.521848] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 4371.521850] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 4371.521851] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 4371.521852] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 4371.521853] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 4371.521856] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 4371.521857] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4371.521858] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 4371.521859] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4371.521860] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4371.521861] PKRU: 55555554 [ 4371.521862] Call Trace: [ 4371.521864] [ 4371.521867] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 4371.521872] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 4371.521878] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 4371.521880] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 4371.521885] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 4371.521887] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 4371.521892] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4371.521895] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4371.521897] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4371.521900] [ 4371.521900] [ 4371.521901] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4371.521907] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4371.521909] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4371.521913] ? up_read (kernel/locking/rwsem.c:1622) [ 4371.521917] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4371.521921] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4371.521923] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4371.521925] __tcf_action_put (net/sched/act_api.c:377) [ 4371.521930] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4371.521932] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4371.521935] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4371.521942] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4371.521948] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4371.521954] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4371.521957] tcf_chain_flush (net/sched/cls_api.c:737) [ 4371.521958] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4371.521960] tcf_block_put (net/sched/cls_api.c:1543) [ 4371.521963] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4371.521967] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4371.521972] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4371.521975] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 4371.521980] default_device_exit_batch (net/core/dev.c:11592) [ 4371.521983] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4371.521985] ? process_one_work (kernel/workqueue.c:2606) [ 4371.521990] process_one_work (kernel/workqueue.c:2633) [ 4371.521993] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4371.521995] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4371.521997] kthread (kernel/kthread.c:388) [ 4371.522000] ? __pfx_kthread (kernel/kthread.c:341) [ 4371.522002] ret_from_fork (arch/x86/kernel/process.c:147) [ 4371.522006] ? __pfx_kthread (kernel/kthread.c:341) [ 4371.522007] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4371.522013] | [ 4406.253609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4406.253940] rcu: 3-....: (4393593 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=1013658 | [ 4406.254462] rcu: (t=4395284 jiffies g=7477 q=1777 ncpus=4) [ 4406.255214] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4406.255810] Workqueue: netns cleanup_net [ 4406.256025] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4406.256318] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4406.257287] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 4406.257564] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 4406.257948] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 4406.258329] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 4406.258711] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 4406.259095] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 4406.259477] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 4406.259905] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4406.260216] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 4406.260600] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4406.260983] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4406.261363] PKRU: 55555554 [ 4406.261515] Call Trace: [ 4406.261655] [ 4406.261774] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4406.262018] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4406.262269] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 4406.262484] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 4406.262700] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 4406.262911] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4406.263154] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4406.263398] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4406.263686] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4406.263912] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4406.264172] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4406.264414] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4406.264655] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4406.264939] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4406.265209] [ 4406.265331] [ 4406.265453] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4406.265743] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4406.266009] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4406.266221] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4406.266425] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4406.266699] ? up_read (kernel/locking/rwsem.c:1622) [ 4406.266881] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4406.267126] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4406.267396] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4406.267657] __tcf_action_put (net/sched/act_api.c:377) [ 4406.267869] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4406.268088] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4406.268302] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4406.268591] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4406.268853] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4406.269104] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4406.269323] tcf_chain_flush (net/sched/cls_api.c:737) [ 4406.269534] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4406.269747] tcf_block_put (net/sched/cls_api.c:1543) [ 4406.269945] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4406.270191] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4406.270404] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4406.270603] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 4406.270898] default_device_exit_batch (net/core/dev.c:11592) [ 4406.271160] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4406.271358] ? process_one_work (kernel/workqueue.c:2606) [ 4406.271590] process_one_work (kernel/workqueue.c:2633) [ 4406.271810] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4406.272015] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4406.272251] kthread (kernel/kthread.c:388) [ 4406.272428] ? __pfx_kthread (kernel/kthread.c:341) [ 4406.272635] ret_from_fork (arch/x86/kernel/process.c:147) [ 4406.272835] ? __pfx_kthread (kernel/kthread.c:341) [ 4406.273041] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4469.824631] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 4458825 jiffies s: 4157 root: 0x8/. | [ 4469.825243] rcu: blocking rcu_node structures (internal RCU debug): | [ 4469.825576] Sending NMI from CPU 2 to CPUs 3: | [ 4469.825839] NMI backtrace for cpu 3 [ 4469.825848] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4469.825852] Workqueue: netns cleanup_net [ 4469.825860] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4469.825868] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4469.825869] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 4469.825871] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 4469.825872] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 4469.825873] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 4469.825874] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 4469.825875] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 4469.825877] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 4469.825878] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4469.825879] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 4469.825881] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4469.825881] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4469.825882] PKRU: 55555554 [ 4469.825883] Call Trace: [ 4469.825889] [ 4469.825895] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 4469.825900] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 4469.825905] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 4469.825908] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 4469.825912] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 4469.825914] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 4469.825919] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4469.825922] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4469.825925] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4469.825927] [ 4469.825927] [ 4469.825928] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4469.825934] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4469.825936] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4469.825940] ? up_read (kernel/locking/rwsem.c:1622) [ 4469.825944] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4469.825948] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4469.825949] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4469.825952] __tcf_action_put (net/sched/act_api.c:377) [ 4469.825956] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4469.825958] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4469.825961] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4469.825967] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4469.825973] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4469.825978] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4469.825981] tcf_chain_flush (net/sched/cls_api.c:737) [ 4469.825983] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4469.825986] tcf_block_put (net/sched/cls_api.c:1543) [ 4469.825988] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4469.825993] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4469.825998] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4469.826000] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 4469.826005] default_device_exit_batch (net/core/dev.c:11592) [ 4469.826008] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4469.826011] ? process_one_work (kernel/workqueue.c:2606) [ 4469.826015] process_one_work (kernel/workqueue.c:2633) [ 4469.826018] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4469.826020] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4469.826022] kthread (kernel/kthread.c:388) [ 4469.826025] ? __pfx_kthread (kernel/kthread.c:341) [ 4469.826027] ret_from_fork (arch/x86/kernel/process.c:147) [ 4469.826031] ? __pfx_kthread (kernel/kthread.c:341) [ 4469.826033] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4469.826038] | [ 4484.276609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4484.276941] rcu: 3-....: (4471583 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=1033043 | [ 4484.277465] rcu: (t=4473307 jiffies g=7477 q=1777 ncpus=4) [ 4484.278209] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4484.278804] Workqueue: netns cleanup_net [ 4484.279019] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4484.279314] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4484.280278] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 4484.280556] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 4484.280930] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 4484.281308] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 4484.281685] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 4484.282058] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 4484.282433] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 4484.282859] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4484.283162] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 4484.283537] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4484.283912] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4484.284290] PKRU: 55555554 [ 4484.284438] Call Trace: [ 4484.284574] [ 4484.284692] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4484.284933] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4484.285175] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 4484.285391] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 4484.285599] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 4484.285807] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4484.286048] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4484.286283] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4484.286569] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4484.286788] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4484.287045] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4484.287286] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4484.287516] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4484.287795] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4484.288061] [ 4484.288178] [ 4484.288296] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4484.288581] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4484.288844] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4484.289053] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4484.289252] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4484.289519] ? up_read (kernel/locking/rwsem.c:1622) [ 4484.289698] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4484.289938] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4484.290203] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4484.290460] __tcf_action_put (net/sched/act_api.c:377) [ 4484.290670] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4484.290887] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4484.291094] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4484.291374] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4484.291631] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4484.291878] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4484.292091] tcf_chain_flush (net/sched/cls_api.c:737) [ 4484.292296] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4484.292505] tcf_block_put (net/sched/cls_api.c:1543) [ 4484.292701] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4484.292943] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4484.293151] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4484.293349] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 4484.293640] default_device_exit_batch (net/core/dev.c:11592) [ 4484.293898] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4484.294094] ? process_one_work (kernel/workqueue.c:2606) [ 4484.294323] process_one_work (kernel/workqueue.c:2633) [ 4484.294541] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4484.294747] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4484.294977] kthread (kernel/kthread.c:388) [ 4484.295151] ? __pfx_kthread (kernel/kthread.c:341) [ 4484.295354] ret_from_fork (arch/x86/kernel/process.c:147) [ 4484.295550] ? __pfx_kthread (kernel/kthread.c:341) [ 4484.295757] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4484.295972] | [ 4562.299610] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4562.299956] rcu: 3-....: (4549588 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=1051714 | [ 4562.300481] rcu: (t=4551330 jiffies g=7477 q=1777 ncpus=4) [ 4562.301229] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4562.301825] Workqueue: netns cleanup_net [ 4562.302046] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4562.302342] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4562.303304] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 4562.303580] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 4562.303960] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 4562.304336] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 4562.304712] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 4562.305085] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 4562.305462] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 4562.305887] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4562.306194] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 4562.306580] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4562.306972] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4562.307357] PKRU: 55555554 [ 4562.307509] Call Trace: [ 4562.307650] [ 4562.307772] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4562.308013] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4562.308257] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 4562.308472] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 4562.308683] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 4562.308889] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4562.309129] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4562.309363] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4562.309651] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4562.309869] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4562.310125] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4562.310367] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4562.310596] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4562.310877] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4562.311143] [ 4562.311260] [ 4562.311378] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4562.311666] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4562.311928] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4562.312137] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4562.312336] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4562.312604] ? up_read (kernel/locking/rwsem.c:1622) [ 4562.312784] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4562.313024] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4562.313289] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4562.313545] __tcf_action_put (net/sched/act_api.c:377) [ 4562.313758] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4562.313980] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4562.314187] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4562.314469] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4562.314726] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4562.314974] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4562.315186] tcf_chain_flush (net/sched/cls_api.c:737) [ 4562.315392] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4562.315600] tcf_block_put (net/sched/cls_api.c:1543) [ 4562.315798] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4562.316038] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4562.316248] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4562.316443] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 4562.316736] default_device_exit_batch (net/core/dev.c:11592) [ 4562.316993] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4562.317189] ? process_one_work (kernel/workqueue.c:2606) [ 4562.317414] process_one_work (kernel/workqueue.c:2633) [ 4562.317633] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4562.317837] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4562.318066] kthread (kernel/kthread.c:388) [ 4562.318241] ? __pfx_kthread (kernel/kthread.c:341) [ 4562.318444] ret_from_fork (arch/x86/kernel/process.c:147) [ 4562.318641] ? __pfx_kthread (kernel/kthread.c:341) [ 4562.318843] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4568.128619] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 4557129 jiffies s: 4157 root: 0x8/. | [ 4568.129232] rcu: blocking rcu_node structures (internal RCU debug): | [ 4568.129575] Sending NMI from CPU 2 to CPUs 3: | [ 4568.129825] NMI backtrace for cpu 3 [ 4568.129830] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4568.129831] Workqueue: netns cleanup_net [ 4568.129834] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4568.129837] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4568.129838] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 4568.129839] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 4568.129840] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 4568.129841] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 4568.129841] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 4568.129842] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 4568.129846] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 4568.129847] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4568.129847] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 4568.129848] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4568.129849] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4568.129850] PKRU: 55555554 [ 4568.129850] Call Trace: [ 4568.129851] [ 4568.129852] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 4568.129857] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 4568.129860] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 4568.129863] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 4568.129867] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 4568.129869] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 4568.129873] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4568.129875] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4568.129877] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4568.129880] [ 4568.129880] [ 4568.129881] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4568.129883] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4568.129885] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4568.129887] ? up_read (kernel/locking/rwsem.c:1622) [ 4568.129889] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4568.129892] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4568.129894] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4568.129896] __tcf_action_put (net/sched/act_api.c:377) [ 4568.129898] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4568.129900] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4568.129902] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4568.129906] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4568.129912] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4568.129917] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4568.129919] tcf_chain_flush (net/sched/cls_api.c:737) [ 4568.129920] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4568.129922] tcf_block_put (net/sched/cls_api.c:1543) [ 4568.129924] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4568.129927] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4568.129930] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4568.129932] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 4568.129934] default_device_exit_batch (net/core/dev.c:11592) [ 4568.129936] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4568.129939] ? process_one_work (kernel/workqueue.c:2606) [ 4568.129940] process_one_work (kernel/workqueue.c:2633) [ 4568.129943] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4568.129945] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4568.129947] kthread (kernel/kthread.c:388) [ 4568.129948] ? __pfx_kthread (kernel/kthread.c:341) [ 4568.129950] ret_from_fork (arch/x86/kernel/process.c:147) [ 4568.129953] ? __pfx_kthread (kernel/kthread.c:341) [ 4568.129954] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4568.129958] | [ 4640.322609] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4640.322946] rcu: 3-....: (4627578 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=1068908 | [ 4640.323475] rcu: (t=4629353 jiffies g=7477 q=1777 ncpus=4) [ 4640.324230] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4640.324828] Workqueue: netns cleanup_net [ 4640.325050] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 4640.325347] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 <85> c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: f3 0f 1e fa endbr64 12: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 17: 41 55 push %r13 19: 41 54 push %r12 1b: 55 push %rbp 1c: 48 89 fd mov %rdi,%rbp 1f: 53 push %rbx 20: 66 90 xchg %ax,%ax 22: ba 01 00 00 00 mov $0x1,%edx 27: 8b 45 00 mov 0x0(%rbp),%eax 2a:* 85 c0 test %eax,%eax <-- trapping instruction 2c: 75 14 jne 0x42 2e: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 33: 85 c0 test %eax,%eax 35: 75 f0 jne 0x27 37: 5b pop %rbx 38: 5d pop %rbp 39: 41 5c pop %r12 3b: 41 5d pop %r13 3d: c3 ret 3e: cc int3 3f: cc int3 Code starting with the faulting instruction =========================================== 0: 85 c0 test %eax,%eax 2: 75 14 jne 0x18 4: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 9: 85 c0 test %eax,%eax b: 75 f0 jne 0xfffffffffffffffd d: 5b pop %rbx e: 5d pop %rbp f: 41 5c pop %r12 11: 41 5d pop %r13 13: c3 ret 14: cc int3 15: cc int3 [ 4640.326313] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 4640.326595] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 4640.326973] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 4640.327346] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 4640.327724] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 4640.328099] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 4640.328476] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 4640.328902] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4640.329207] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 4640.329583] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4640.329958] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4640.330334] PKRU: 55555554 [ 4640.330482] Call Trace: [ 4640.330623] [ 4640.330747] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4640.330988] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4640.331234] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 4640.331447] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 4640.331660] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 4640.331868] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4640.332105] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4640.332344] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4640.332628] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4640.332851] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4640.333112] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4640.333351] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4640.333589] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4640.333873] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4640.334137] [ 4640.334256] [ 4640.334375] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4640.334661] ? queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 4640.334924] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4640.335132] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4640.335332] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4640.335601] ? up_read (kernel/locking/rwsem.c:1622) [ 4640.335780] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4640.336024] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4640.336288] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4640.336546] __tcf_action_put (net/sched/act_api.c:377) [ 4640.336760] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4640.336979] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4640.337188] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4640.337470] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4640.337728] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4640.337977] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4640.338189] tcf_chain_flush (net/sched/cls_api.c:737) [ 4640.338396] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4640.338604] tcf_block_put (net/sched/cls_api.c:1543) [ 4640.338802] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4640.339046] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4640.339257] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4640.339453] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 4640.339744] default_device_exit_batch (net/core/dev.c:11592) [ 4640.340002] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4640.340198] ? process_one_work (kernel/workqueue.c:2606) [ 4640.340426] process_one_work (kernel/workqueue.c:2633) [ 4640.340643] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4640.340847] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4640.341080] kthread (kernel/kthread.c:388) [ 4640.341255] ? __pfx_kthread (kernel/kthread.c:341) [ 4640.341458] ret_from_fork (arch/x86/kernel/process.c:147) [ 4640.341657] ? __pfx_kthread (kernel/kthread.c:341) [ 4640.341863] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4666.432618] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 4655433 jiffies s: 4157 root: 0x8/. | [ 4666.433230] rcu: blocking rcu_node structures (internal RCU debug): | [ 4666.433566] Sending NMI from CPU 2 to CPUs 3: | [ 4666.433816] NMI backtrace for cpu 3 [ 4666.433823] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4666.433825] Workqueue: netns cleanup_net [ 4666.433831] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4666.433838] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4666.433839] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 4666.433840] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 4666.433841] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 4666.433842] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 4666.433843] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 4666.433844] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 4666.433847] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 4666.433848] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4666.433849] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 4666.433850] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4666.433850] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4666.433851] PKRU: 55555554 [ 4666.433852] Call Trace: [ 4666.433854] [ 4666.433856] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 4666.433861] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 4666.433865] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 4666.433868] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 4666.433872] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 4666.433874] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 4666.433878] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4666.433881] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4666.433883] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4666.433885] [ 4666.433886] [ 4666.433886] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4666.433891] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4666.433893] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4666.433897] ? up_read (kernel/locking/rwsem.c:1622) [ 4666.433900] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4666.433904] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4666.433906] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4666.433908] __tcf_action_put (net/sched/act_api.c:377) [ 4666.433912] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4666.433914] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4666.433917] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4666.433923] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4666.433929] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4666.433934] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4666.433936] tcf_chain_flush (net/sched/cls_api.c:737) [ 4666.433938] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4666.433940] tcf_block_put (net/sched/cls_api.c:1543) [ 4666.433941] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4666.433946] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4666.433950] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4666.433952] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 4666.433956] default_device_exit_batch (net/core/dev.c:11592) [ 4666.433959] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4666.433961] ? process_one_work (kernel/workqueue.c:2606) [ 4666.433965] process_one_work (kernel/workqueue.c:2633) [ 4666.433968] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4666.433970] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4666.433972] kthread (kernel/kthread.c:388) [ 4666.433974] ? __pfx_kthread (kernel/kthread.c:341) [ 4666.433976] ret_from_fork (arch/x86/kernel/process.c:147) [ 4666.433980] ? __pfx_kthread (kernel/kthread.c:341) [ 4666.433982] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4666.433986] | [ 4718.344610] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4718.344951] rcu: 3-....: (4705567 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=1086444 | [ 4718.345473] rcu: (t=4707375 jiffies g=7477 q=1777 ncpus=4) [ 4718.346219] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4718.346816] Workqueue: netns cleanup_net [ 4718.347034] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4718.347331] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4718.348290] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 4718.348569] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 4718.348942] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 4718.349316] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 4718.349692] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 4718.350066] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 4718.350439] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 4718.350865] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4718.351170] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 4718.351542] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4718.351918] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4718.352294] PKRU: 55555554 [ 4718.352441] Call Trace: [ 4718.352577] [ 4718.352695] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4718.352936] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4718.353188] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 4718.353404] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 4718.353613] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 4718.353820] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4718.354059] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4718.354293] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4718.354576] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4718.354795] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4718.355051] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4718.355291] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4718.355523] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4718.355803] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4718.356066] [ 4718.356183] [ 4718.356300] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4718.356586] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4718.356848] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4718.357056] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4718.357255] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4718.357520] ? up_read (kernel/locking/rwsem.c:1622) [ 4718.357701] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4718.357941] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4718.358205] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4718.358461] __tcf_action_put (net/sched/act_api.c:377) [ 4718.358671] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4718.358886] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4718.359092] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4718.359372] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4718.359628] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4718.359875] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4718.360086] tcf_chain_flush (net/sched/cls_api.c:737) [ 4718.360293] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4718.360501] tcf_block_put (net/sched/cls_api.c:1543) [ 4718.360696] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4718.360936] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4718.361145] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4718.361339] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 4718.361629] default_device_exit_batch (net/core/dev.c:11592) [ 4718.361886] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4718.362082] ? process_one_work (kernel/workqueue.c:2606) [ 4718.362306] process_one_work (kernel/workqueue.c:2633) [ 4718.362524] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4718.362726] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4718.362955] kthread (kernel/kthread.c:388) [ 4718.363134] ? __pfx_kthread (kernel/kthread.c:341) [ 4718.363336] ret_from_fork (arch/x86/kernel/process.c:147) [ 4718.363531] ? __pfx_kthread (kernel/kthread.c:341) [ 4718.363738] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4764.736620] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 4753737 jiffies s: 4157 root: 0x8/. | [ 4764.737237] rcu: blocking rcu_node structures (internal RCU debug): | [ 4764.737568] Sending NMI from CPU 2 to CPUs 3: | [ 4764.737820] NMI backtrace for cpu 3 [ 4764.737828] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4764.737830] Workqueue: netns cleanup_net [ 4764.737838] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4764.737844] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4764.737846] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 4764.737847] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 4764.737848] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 4764.737849] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 4764.737850] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 4764.737851] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 4764.737855] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 4764.737856] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4764.737856] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 4764.737857] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4764.737858] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4764.737859] PKRU: 55555554 [ 4764.737859] Call Trace: [ 4764.737862] [ 4764.737864] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 4764.737870] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 4764.737874] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 4764.737877] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 4764.737881] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 4764.737883] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 4764.737888] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4764.737890] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4764.737892] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4764.737895] [ 4764.737895] [ 4764.737896] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4764.737901] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4764.737902] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4764.737907] ? up_read (kernel/locking/rwsem.c:1622) [ 4764.737911] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4764.737915] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4764.737917] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4764.737919] __tcf_action_put (net/sched/act_api.c:377) [ 4764.737924] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4764.737926] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4764.737928] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4764.737935] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4764.737941] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4764.737946] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4764.737949] tcf_chain_flush (net/sched/cls_api.c:737) [ 4764.737951] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4764.737953] tcf_block_put (net/sched/cls_api.c:1543) [ 4764.737956] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4764.737960] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4764.737965] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4764.737967] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 4764.737971] default_device_exit_batch (net/core/dev.c:11592) [ 4764.737974] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4764.737977] ? process_one_work (kernel/workqueue.c:2606) [ 4764.737981] process_one_work (kernel/workqueue.c:2633) [ 4764.737984] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4764.737987] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4764.737989] kthread (kernel/kthread.c:388) [ 4764.737992] ? __pfx_kthread (kernel/kthread.c:341) [ 4764.737993] ret_from_fork (arch/x86/kernel/process.c:147) [ 4764.737998] ? __pfx_kthread (kernel/kthread.c:341) [ 4764.737999] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4764.738005] | [ 4796.367608] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4796.367948] rcu: 3-....: (4783557 ticks this GP) idle=bd64/1/0x4000000000000000 softirq=5764/5764 fqs=1103965 | [ 4796.368470] rcu: (t=4785398 jiffies g=7477 q=1777 ncpus=4) [ 4796.369214] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4796.369808] Workqueue: netns cleanup_net [ 4796.370026] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4796.370319] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4796.371283] RSP: 0018:ffff9e38400639d8 EFLAGS: 00000282 [ 4796.371559] RAX: 0000000082fc8b08 RBX: ffff906783ad2708 RCX: 0000000000000000 [ 4796.371934] RDX: 0000000000000001 RSI: 0000000082fc8b08 RDI: ffff906783ad2708 [ 4796.372304] RBP: ffff906783ad2708 R08: 0000000000000001 R09: 0000000000000000 [ 4796.372679] R10: 0000000000000000 R11: ffff9e3840317cc8 R12: ffff906783ad2700 [ 4796.373058] R13: 0000000000000000 R14: ffff906783ad2708 R15: ffff906783ad2768 [ 4796.373431] FS: 0000000000000000(0000) GS:ffff9067bed80000(0000) knlGS:0000000000000000 [ 4796.373852] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4796.374156] CR2: 00007f3d53919a30 CR3: 0000000039660001 CR4: 0000000000770ef0 [ 4796.374529] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4796.374902] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4796.375274] PKRU: 55555554 [ 4796.375422] Call Trace: [ 4796.375559] [ 4796.375674] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4796.375913] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4796.376159] ? cpuacct_charge (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:341) [ 4796.376371] ? lock_release (./include/trace/events/lock.h:69 kernel/locking/lockdep.c:5765) [ 4796.376578] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) [ 4796.376787] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4796.377026] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4796.377263] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4796.377547] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4796.377769] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4796.378027] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4796.378270] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4796.378503] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4796.378783] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4796.379051] [ 4796.379173] [ 4796.379295] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4796.379582] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4796.379848] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4796.380059] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4796.380263] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4796.380533] ? up_read (kernel/locking/rwsem.c:1622) [ 4796.380716] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4796.380960] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4796.381231] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4796.381490] __tcf_action_put (net/sched/act_api.c:377) [ 4796.381702] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4796.381922] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4796.382132] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4796.382420] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4796.382679] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4796.382931] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4796.383148] tcf_chain_flush (net/sched/cls_api.c:737) [ 4796.383358] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4796.383567] tcf_block_put (net/sched/cls_api.c:1543) [ 4796.383765] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4796.384009] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4796.384221] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4796.384419] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11047) [ 4796.384712] default_device_exit_batch (net/core/dev.c:11592) [ 4796.384973] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4796.385172] ? process_one_work (kernel/workqueue.c:2606) [ 4796.385401] process_one_work (kernel/workqueue.c:2633) [ 4796.385622] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4796.385828] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4796.386065] kthread (kernel/kthread.c:388) [ 4796.386240] ? __pfx_kthread (kernel/kthread.c:341) [ 4796.386445] ret_from_fork (arch/x86/kernel/process.c:147) [ 4796.386643] ? __pfx_kthread (kernel/kthread.c:341) Finger prints: __mutex_lock:refcount_dec_and_mutex_lock:__tcf_action_put:tcf_action_destroy queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock