====================================== | [ 43.342225] ------------[ cut here ]------------ | [ 43.342805] DEBUG_LOCKS_WARN_ON(lock->magic != lock) | [ 43.342819] WARNING: CPU: 1 PID: 11 at kernel/locking/mutex.c:587 __mutex_lock (kernel/locking/mutex.c:587 kernel/locking/mutex.c:752) | [ 43.344330] Modules linked in: act_csum libcrc32c act_pedit cls_flower sch_prio bonding xfrm_user psample macsec vxlan ip6_gre ip_gre gre cls_u32 sch_htb [last unloaded: netdevsim] [ 43.347039] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 43.348314] Workqueue: netns cleanup_net [ 43.348697] RIP: 0010:__mutex_lock (kernel/locking/mutex.c:587 kernel/locking/mutex.c:752) [ 43.349213] Code: ff 90 e8 a2 e7 70 ff 85 c0 74 23 8b 35 9c 8e db 00 85 f6 75 19 90 48 c7 c6 97 37 aa 8f 48 c7 c7 3a ac a9 8f e8 00 04 26 ff 90 <0f> 0b 90 90 90 e9 03 fd ff ff 4c 8d 6d a0 4c 89 e7 4c 89 ee e8 d6 All code ======== 0: ff 90 e8 a2 e7 70 call *0x70e7a2e8(%rax) 6: ff 85 c0 74 23 8b incl -0x74dc8b40(%rbp) c: 35 9c 8e db 00 xor $0xdb8e9c,%eax 11: 85 f6 test %esi,%esi 13: 75 19 jne 0x2e 15: 90 nop 16: 48 c7 c6 97 37 aa 8f mov $0xffffffff8faa3797,%rsi 1d: 48 c7 c7 3a ac a9 8f mov $0xffffffff8fa9ac3a,%rdi 24: e8 00 04 26 ff call 0xffffffffff260429 29: 90 nop 2a:* 0f 0b ud2 <-- trapping instruction 2c: 90 nop 2d: 90 nop 2e: 90 nop 2f: e9 03 fd ff ff jmp 0xfffffffffffffd37 34: 4c 8d 6d a0 lea -0x60(%rbp),%r13 38: 4c 89 e7 mov %r12,%rdi 3b: 4c 89 ee mov %r13,%rsi 3e: e8 .byte 0xe8 3f: d6 (bad) Code starting with the faulting instruction =========================================== 0: 0f 0b ud2 2: 90 nop 3: 90 nop 4: 90 nop 5: e9 03 fd ff ff jmp 0xfffffffffffffd0d a: 4c 8d 6d a0 lea -0x60(%rbp),%r13 e: 4c 89 e7 mov %r12,%rdi 11: 4c 89 ee mov %r13,%rsi 14: e8 .byte 0xe8 15: d6 (bad) [ 43.351126] RSP: 0018:ffffb56500063a10 EFLAGS: 00010286 [ 43.351651] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 00000000ffffdfff [ 43.352195] RDX: 0000000000000000 RSI: 00000000ffffffea RDI: 0000000000000001 [ 43.352720] RBP: ffffb56500063a90 R08: ffffffff8ff78508 R09: 00000000ffffdfff [ 43.353302] R10: ffffffff8fe98520 R11: ffffffff8ff4b070 R12: ffff9b6b02518900 [ 43.353958] R13: 0000000000000000 R14: 0000000000000000 R15: ffff9b6b02aa1400 [ 43.354347] FS: 0000000000000000(0000) GS:ffff9b6b3ec80000(0000) knlGS:0000000000000000 [ 43.355100] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 43.355529] CR2: 00007fc50215f000 CR3: 0000000025460006 CR4: 0000000000770ef0 [ 43.356344] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 43.357381] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 43.358452] PKRU: 55555554 [ 43.358889] Call Trace: [ 43.359269] [ 43.359627] ? __warn (kernel/panic.c:677) [ 43.360147] ? __mutex_lock (kernel/locking/mutex.c:587 kernel/locking/mutex.c:752) [ 43.360881] ? report_bug (lib/bug.c:201 lib/bug.c:219) [ 43.361388] ? handle_bug (arch/x86/kernel/traps.c:238) [ 43.361952] ? exc_invalid_op (arch/x86/kernel/traps.c:259 (discriminator 1)) [ 43.362435] ? asm_exc_invalid_op (./arch/x86/include/asm/idtentry.h:568) [ 43.362954] ? __mutex_lock (kernel/locking/mutex.c:587 kernel/locking/mutex.c:752) [ 43.363458] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 43.364038] ? up_read (kernel/locking/rwsem.c:1622) [ 43.364463] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 43.365026] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 43.365587] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 43.365875] __tcf_action_put (net/sched/act_api.c:377) [ 43.366134] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 43.366383] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 43.366598] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 43.366900] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 43.367169] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 43.367428] tcf_proto_destroy (net/sched/cls_api.c:418) [ 43.367655] tcf_chain_flush (net/sched/cls_api.c:737) [ 43.367878] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 43.368099] tcf_block_put (net/sched/cls_api.c:1543) [ 43.368305] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 43.368557] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 43.368777] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 43.368989] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 43.369290] default_device_exit_batch (net/core/dev.c:11629) [ 43.369559] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 43.369780] ? process_one_work (kernel/workqueue.c:2606) [ 43.370020] process_one_work (kernel/workqueue.c:2633) [ 43.370244] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 43.370452] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 43.370689] kthread (kernel/kthread.c:388) [ 43.370873] ? __pfx_kthread (kernel/kthread.c:341) [ 43.371084] ret_from_fork (arch/x86/kernel/process.c:147) [ 43.371286] ? __pfx_kthread (kernel/kthread.c:341) [ 43.371494] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 43.374209] ---[ end trace 0000000000000000 ]--- | [ 69.374865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 69.375202] rcu: 3-....: (25999 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=6425 | [ 69.375711] rcu: (t=26000 jiffies g=31081 q=2267 ncpus=4) [ 69.376453] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 69.377049] Workqueue: netns cleanup_net [ 69.377266] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 69.377562] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 69.378525] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 69.378802] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 69.379179] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 69.379553] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 69.379929] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 69.380303] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 69.380680] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 69.381106] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 69.381410] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 69.381787] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 69.382163] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 69.382538] PKRU: 55555554 [ 69.382686] Call Trace: [ 69.382822] [ 69.382939] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 69.383179] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 69.383423] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 69.383694] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 69.383937] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 69.384181] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 69.384413] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 69.384698] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 69.384918] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 69.385173] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 69.385415] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 69.385644] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 69.385924] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 69.386191] [ 69.386308] [ 69.386426] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 69.386711] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 69.386972] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 69.387184] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 69.387386] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 69.387655] ? up_read (kernel/locking/rwsem.c:1622) [ 69.387835] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 69.388076] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 69.388342] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 69.388598] __tcf_action_put (net/sched/act_api.c:377) [ 69.388812] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 69.389028] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 69.389235] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 69.389517] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 69.389772] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 69.390019] tcf_proto_destroy (net/sched/cls_api.c:418) [ 69.390250] tcf_chain_flush (net/sched/cls_api.c:737) [ 69.390457] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 69.390668] tcf_block_put (net/sched/cls_api.c:1543) [ 69.390864] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 69.391105] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 69.391316] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 69.391512] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 69.391802] default_device_exit_batch (net/core/dev.c:11629) [ 69.392060] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 69.392257] ? process_one_work (kernel/workqueue.c:2606) [ 69.392481] process_one_work (kernel/workqueue.c:2633) [ 69.392697] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 69.392903] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 69.393132] kthread (kernel/kthread.c:388) [ 69.393308] ? __pfx_kthread (kernel/kthread.c:341) [ 69.393511] ret_from_fork (arch/x86/kernel/process.c:147) [ 69.393722] ? __pfx_kthread (kernel/kthread.c:341) [ 69.393925] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 78.909875] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 35501 jiffies s: 6353 root: 0x8/. | [ 78.910470] rcu: blocking rcu_node structures (internal RCU debug): | [ 78.910796] Sending NMI from CPU 1 to CPUs 3: | [ 78.911049] NMI backtrace for cpu 3 [ 78.911054] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 78.911056] Workqueue: netns cleanup_net [ 78.911062] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 78.911068] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 78.911069] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 78.911071] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 78.911072] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 78.911073] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 78.911074] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 78.911075] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 78.911077] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 78.911078] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 78.911079] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 78.911080] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 78.911081] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 78.911081] PKRU: 55555554 [ 78.911082] Call Trace: [ 78.911084] [ 78.911086] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 78.911089] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 78.911093] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 78.911095] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 78.911098] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 78.911100] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 78.911104] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 78.911106] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 78.911109] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 78.911110] [ 78.911111] [ 78.911112] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 78.911116] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 78.911120] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 78.911123] ? up_read (kernel/locking/rwsem.c:1622) [ 78.911127] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 78.911131] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 78.911132] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 78.911135] __tcf_action_put (net/sched/act_api.c:377) [ 78.911140] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 78.911142] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 78.911144] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 78.911150] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 78.911156] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 78.911161] tcf_proto_destroy (net/sched/cls_api.c:418) [ 78.911164] tcf_chain_flush (net/sched/cls_api.c:737) [ 78.911166] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 78.911169] tcf_block_put (net/sched/cls_api.c:1543) [ 78.911173] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 78.911177] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 78.911181] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 78.911183] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 78.911187] default_device_exit_batch (net/core/dev.c:11629) [ 78.911189] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 78.911192] ? process_one_work (kernel/workqueue.c:2606) [ 78.911195] process_one_work (kernel/workqueue.c:2633) [ 78.911198] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 78.911200] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 78.911202] kthread (kernel/kthread.c:388) [ 78.911205] ? __pfx_kthread (kernel/kthread.c:341) [ 78.911206] ret_from_fork (arch/x86/kernel/process.c:147) [ 78.911209] ? __pfx_kthread (kernel/kthread.c:341) [ 78.911210] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 78.911217] | [ 147.392865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 147.393214] rcu: 3-....: (103984 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=25741 | [ 147.393740] rcu: (t=104018 jiffies g=31081 q=2270 ncpus=4) [ 147.394490] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 147.395087] Workqueue: netns cleanup_net [ 147.395303] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 147.395602] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 147.396564] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 147.396846] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 147.397223] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 147.397599] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 147.397978] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 147.398356] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 147.398732] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 147.399159] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 147.399466] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 147.399842] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 147.400219] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 147.400595] PKRU: 55555554 [ 147.400744] Call Trace: [ 147.400882] [ 147.401000] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 147.401242] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 147.401488] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 147.401756] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 147.401996] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 147.402242] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 147.402481] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 147.402766] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 147.402985] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 147.403252] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 147.403493] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 147.403726] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 147.404008] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 147.404277] [ 147.404396] [ 147.404515] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 147.404799] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 147.405062] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 147.405273] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 147.405475] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 147.405743] ? up_read (kernel/locking/rwsem.c:1622) [ 147.405924] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 147.406167] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 147.406432] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 147.406691] __tcf_action_put (net/sched/act_api.c:377) [ 147.406905] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 147.407124] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 147.407331] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 147.407616] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 147.407871] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 147.408118] tcf_proto_destroy (net/sched/cls_api.c:418) [ 147.408333] tcf_chain_flush (net/sched/cls_api.c:737) [ 147.408543] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 147.408752] tcf_block_put (net/sched/cls_api.c:1543) [ 147.408949] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 147.409192] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 147.409402] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 147.409599] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 147.409887] default_device_exit_batch (net/core/dev.c:11629) [ 147.410146] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 147.410342] ? process_one_work (kernel/workqueue.c:2606) [ 147.410569] process_one_work (kernel/workqueue.c:2633) [ 147.410787] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 147.410991] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 147.411222] kthread (kernel/kthread.c:388) [ 147.411397] ? __pfx_kthread (kernel/kthread.c:341) [ 147.411599] ret_from_fork (arch/x86/kernel/process.c:147) [ 147.411796] ? __pfx_kthread (kernel/kthread.c:341) [ 147.411998] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 177.213880] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 133805 jiffies s: 6353 root: 0x8/. | [ 177.214503] rcu: blocking rcu_node structures (internal RCU debug): | [ 177.214841] Sending NMI from CPU 1 to CPUs 3: | [ 177.215095] NMI backtrace for cpu 3 [ 177.215102] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 177.215105] Workqueue: netns cleanup_net [ 177.215113] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 177.215121] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 177.215123] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 177.215124] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 177.215125] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 177.215127] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 177.215128] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 177.215129] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 177.215132] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 177.215133] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 177.215134] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 177.215135] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 177.215136] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 177.215136] PKRU: 55555554 [ 177.215137] Call Trace: [ 177.215140] [ 177.215143] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 177.215147] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 177.215152] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 177.215155] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 177.215160] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 177.215161] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 177.215166] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 177.215168] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 177.215170] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 177.215172] [ 177.215172] [ 177.215173] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 177.215179] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 177.215183] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 177.215188] ? up_read (kernel/locking/rwsem.c:1622) [ 177.215192] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 177.215198] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 177.215199] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 177.215202] __tcf_action_put (net/sched/act_api.c:377) [ 177.215209] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 177.215211] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 177.215214] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 177.215221] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 177.215227] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 177.215232] tcf_proto_destroy (net/sched/cls_api.c:418) [ 177.215235] tcf_chain_flush (net/sched/cls_api.c:737) [ 177.215238] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 177.215241] tcf_block_put (net/sched/cls_api.c:1543) [ 177.215245] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 177.215250] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 177.215254] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 177.215257] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 177.215261] default_device_exit_batch (net/core/dev.c:11629) [ 177.215264] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 177.215267] ? process_one_work (kernel/workqueue.c:2606) [ 177.215272] process_one_work (kernel/workqueue.c:2633) [ 177.215275] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 177.215277] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 177.215279] kthread (kernel/kthread.c:388) [ 177.215282] ? __pfx_kthread (kernel/kthread.c:341) [ 177.215284] ret_from_fork (arch/x86/kernel/process.c:147) [ 177.215287] ? __pfx_kthread (kernel/kthread.c:341) [ 177.215289] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 177.215295] | [ 225.410865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 225.411207] rcu: 3-....: (181969 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=45097 | [ 225.411724] rcu: (t=182036 jiffies g=31081 q=2272 ncpus=4) [ 225.412469] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 225.413067] Workqueue: netns cleanup_net [ 225.413285] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 225.413582] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 225.414539] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 225.414816] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 225.415191] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 225.415563] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 225.415939] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 225.416313] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 225.416689] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 225.417113] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 225.417418] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 225.417791] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 225.418164] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 225.418537] PKRU: 55555554 [ 225.418684] Call Trace: [ 225.418822] [ 225.418938] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 225.419179] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 225.419424] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 225.419689] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 225.419930] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 225.420171] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 225.420404] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 225.420686] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 225.420914] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 225.421170] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 225.421410] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 225.421642] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 225.421920] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 225.422185] [ 225.422304] [ 225.422422] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 225.422704] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 225.422971] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 225.423181] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 225.423380] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 225.423645] ? up_read (kernel/locking/rwsem.c:1622) [ 225.423824] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 225.424067] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 225.424330] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 225.424586] __tcf_action_put (net/sched/act_api.c:377) [ 225.424797] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 225.425012] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 225.425220] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 225.425502] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 225.425757] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 225.426002] tcf_proto_destroy (net/sched/cls_api.c:418) [ 225.426216] tcf_chain_flush (net/sched/cls_api.c:737) [ 225.426423] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 225.426631] tcf_block_put (net/sched/cls_api.c:1543) [ 225.426826] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 225.427069] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 225.427278] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 225.427473] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 225.427759] default_device_exit_batch (net/core/dev.c:11629) [ 225.428016] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 225.428211] ? process_one_work (kernel/workqueue.c:2606) [ 225.428440] process_one_work (kernel/workqueue.c:2633) [ 225.428656] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 225.428858] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 225.429089] kthread (kernel/kthread.c:388) [ 225.429262] ? __pfx_kthread (kernel/kthread.c:341) [ 225.429463] ret_from_fork (arch/x86/kernel/process.c:147) [ 225.429659] ? __pfx_kthread (kernel/kthread.c:341) [ 225.429861] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 275.517896] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 232109 jiffies s: 6353 root: 0x8/. | [ 275.518516] rcu: blocking rcu_node structures (internal RCU debug): | [ 275.518853] Sending NMI from CPU 1 to CPUs 3: | [ 275.519118] NMI backtrace for cpu 3 [ 275.519127] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 275.519130] Workqueue: netns cleanup_net [ 275.519137] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 275.519142] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 275.519144] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 275.519145] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 275.519146] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 275.519147] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 275.519148] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 275.519149] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 275.519151] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 275.519153] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 275.519154] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 275.519155] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 275.519155] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 275.519156] PKRU: 55555554 [ 275.519157] Call Trace: [ 275.519164] [ 275.519169] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 275.519174] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 275.519178] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 275.519181] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 275.519185] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 275.519186] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 275.519191] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 275.519193] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 275.519195] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 275.519197] [ 275.519197] [ 275.519198] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 275.519203] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 275.519207] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 275.519211] ? up_read (kernel/locking/rwsem.c:1622) [ 275.519213] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 275.519218] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 275.519219] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 275.519222] __tcf_action_put (net/sched/act_api.c:377) [ 275.519227] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 275.519229] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 275.519232] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 275.519238] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 275.519243] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 275.519248] tcf_proto_destroy (net/sched/cls_api.c:418) [ 275.519251] tcf_chain_flush (net/sched/cls_api.c:737) [ 275.519253] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 275.519256] tcf_block_put (net/sched/cls_api.c:1543) [ 275.519260] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 275.519265] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 275.519268] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 275.519270] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 275.519274] default_device_exit_batch (net/core/dev.c:11629) [ 275.519277] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 275.519279] ? process_one_work (kernel/workqueue.c:2606) [ 275.519282] process_one_work (kernel/workqueue.c:2633) [ 275.519285] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 275.519287] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 275.519289] kthread (kernel/kthread.c:388) [ 275.519290] ? __pfx_kthread (kernel/kthread.c:341) [ 275.519292] ret_from_fork (arch/x86/kernel/process.c:147) [ 275.519295] ? __pfx_kthread (kernel/kthread.c:341) [ 275.519296] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 275.519302] | [ 303.428865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 303.429220] rcu: 3-....: (259954 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=64143 | [ 303.429739] rcu: (t=260054 jiffies g=31081 q=2273 ncpus=4) [ 303.430483] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 303.431074] Workqueue: netns cleanup_net [ 303.431291] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 303.431587] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 303.432550] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 303.432827] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 303.433201] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 303.433572] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 303.433947] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 303.434321] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 303.434694] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 303.435117] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 303.435419] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 303.435793] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 303.436165] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 303.436538] PKRU: 55555554 [ 303.436684] Call Trace: [ 303.436822] [ 303.436937] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 303.437176] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 303.437420] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 303.437684] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 303.437924] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 303.438165] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 303.438400] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 303.438682] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 303.438904] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 303.439159] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 303.439398] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 303.439629] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 303.439906] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 303.440174] [ 303.440293] [ 303.440411] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 303.440694] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 303.440956] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 303.441165] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 303.441365] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 303.441632] ? up_read (kernel/locking/rwsem.c:1622) [ 303.441812] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 303.442056] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 303.442320] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 303.442581] __tcf_action_put (net/sched/act_api.c:377) [ 303.442792] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 303.443006] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 303.443214] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 303.443498] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 303.443753] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 303.443999] tcf_proto_destroy (net/sched/cls_api.c:418) [ 303.444212] tcf_chain_flush (net/sched/cls_api.c:737) [ 303.444420] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 303.444627] tcf_block_put (net/sched/cls_api.c:1543) [ 303.444824] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 303.445067] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 303.445275] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 303.445470] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 303.445756] default_device_exit_batch (net/core/dev.c:11629) [ 303.446014] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 303.446209] ? process_one_work (kernel/workqueue.c:2606) [ 303.446437] process_one_work (kernel/workqueue.c:2633) [ 303.446653] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 303.446856] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 303.447089] kthread (kernel/kthread.c:388) [ 303.447261] ? __pfx_kthread (kernel/kthread.c:341) [ 303.447462] ret_from_fork (arch/x86/kernel/process.c:147) [ 303.447657] ? __pfx_kthread (kernel/kthread.c:341) [ 303.447865] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 373.821876] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 330413 jiffies s: 6353 root: 0x8/. | [ 373.822531] rcu: blocking rcu_node structures (internal RCU debug): | [ 373.822874] Sending NMI from CPU 1 to CPUs 3: | [ 373.823122] NMI backtrace for cpu 3 [ 373.823129] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 373.823131] Workqueue: netns cleanup_net [ 373.823145] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 373.823153] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 373.823154] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 373.823156] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 373.823157] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 373.823158] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 373.823159] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 373.823160] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 373.823162] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 373.823163] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 373.823164] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 373.823165] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 373.823166] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 373.823166] PKRU: 55555554 [ 373.823167] Call Trace: [ 373.823170] [ 373.823173] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 373.823178] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 373.823183] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 373.823186] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 373.823191] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 373.823193] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 373.823198] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 373.823200] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 373.823202] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 373.823204] [ 373.823205] [ 373.823206] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 373.823211] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 373.823215] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 373.823222] ? up_read (kernel/locking/rwsem.c:1622) [ 373.823225] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 373.823230] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 373.823232] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 373.823235] __tcf_action_put (net/sched/act_api.c:377) [ 373.823242] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 373.823244] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 373.823247] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 373.823255] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 373.823260] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 373.823265] tcf_proto_destroy (net/sched/cls_api.c:418) [ 373.823269] tcf_chain_flush (net/sched/cls_api.c:737) [ 373.823271] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 373.823274] tcf_block_put (net/sched/cls_api.c:1543) [ 373.823278] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 373.823284] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 373.823288] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 373.823290] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 373.823295] default_device_exit_batch (net/core/dev.c:11629) [ 373.823298] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 373.823301] ? process_one_work (kernel/workqueue.c:2606) [ 373.823304] process_one_work (kernel/workqueue.c:2633) [ 373.823307] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 373.823309] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 373.823311] kthread (kernel/kthread.c:388) [ 373.823313] ? __pfx_kthread (kernel/kthread.c:341) [ 373.823315] ret_from_fork (arch/x86/kernel/process.c:147) [ 373.823318] ? __pfx_kthread (kernel/kthread.c:341) [ 373.823320] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 373.823326] | [ 381.446865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 381.447210] rcu: 3-....: (337939 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=83547 | [ 381.447736] rcu: (t=338072 jiffies g=31081 q=2282 ncpus=4) [ 381.448492] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 381.449097] Workqueue: netns cleanup_net [ 381.449325] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 381.449620] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 381.450603] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 381.450881] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 381.451262] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 381.451639] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 381.452033] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 381.452426] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 381.452810] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 381.453237] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 381.453542] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 381.453924] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 381.454308] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 381.454690] PKRU: 55555554 [ 381.454839] Call Trace: [ 381.454977] [ 381.455098] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 381.455338] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 381.455595] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 381.455860] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 381.456114] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 381.456354] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 381.456599] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 381.456881] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 381.457110] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 381.457367] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 381.457614] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 381.457847] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 381.458128] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 381.458396] [ 381.458514] [ 381.458632] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 381.458914] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 381.459178] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 381.459387] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 381.459594] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 381.459861] ? up_read (kernel/locking/rwsem.c:1622) [ 381.460040] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 381.460282] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 381.460545] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 381.460810] __tcf_action_put (net/sched/act_api.c:377) [ 381.461023] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 381.461238] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 381.461458] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 381.461744] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 381.462023] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 381.462288] tcf_proto_destroy (net/sched/cls_api.c:418) [ 381.462511] tcf_chain_flush (net/sched/cls_api.c:737) [ 381.462721] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 381.462935] tcf_block_put (net/sched/cls_api.c:1543) [ 381.463145] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 381.463397] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 381.463607] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 381.463813] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 381.464102] default_device_exit_batch (net/core/dev.c:11629) [ 381.464372] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 381.464570] ? process_one_work (kernel/workqueue.c:2606) [ 381.464805] process_one_work (kernel/workqueue.c:2633) [ 381.465025] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 381.465233] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 381.465476] kthread (kernel/kthread.c:388) [ 381.465652] ? __pfx_kthread (kernel/kthread.c:341) [ 381.465865] ret_from_fork (arch/x86/kernel/process.c:147) [ 381.466063] ? __pfx_kthread (kernel/kthread.c:341) [ 381.466271] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 381.466488] | [ 459.464865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 459.465216] rcu: 3-....: (415939 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=102887 | [ 459.465741] rcu: (t=416090 jiffies g=31081 q=2282 ncpus=4) [ 459.466491] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 459.467112] Workqueue: netns cleanup_net [ 459.467328] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 459.467625] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 <85> c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: f3 0f 1e fa endbr64 12: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 17: 41 55 push %r13 19: 41 54 push %r12 1b: 55 push %rbp 1c: 48 89 fd mov %rdi,%rbp 1f: 53 push %rbx 20: 66 90 xchg %ax,%ax 22: ba 01 00 00 00 mov $0x1,%edx 27: 8b 45 00 mov 0x0(%rbp),%eax 2a:* 85 c0 test %eax,%eax <-- trapping instruction 2c: 75 14 jne 0x42 2e: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 33: 85 c0 test %eax,%eax 35: 75 f0 jne 0x27 37: 5b pop %rbx 38: 5d pop %rbp 39: 41 5c pop %r12 3b: 41 5d pop %r13 3d: c3 ret 3e: cc int3 3f: cc int3 Code starting with the faulting instruction =========================================== 0: 85 c0 test %eax,%eax 2: 75 14 jne 0x18 4: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 9: 85 c0 test %eax,%eax b: 75 f0 jne 0xfffffffffffffffd d: 5b pop %rbx e: 5d pop %rbp f: 41 5c pop %r12 11: 41 5d pop %r13 13: c3 ret 14: cc int3 15: cc int3 [ 459.468580] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 459.468857] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 459.469230] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 459.469603] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 459.469978] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 459.470350] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 459.470725] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 459.471174] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 459.471477] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 459.471862] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 459.472236] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 459.472610] PKRU: 55555554 [ 459.472757] Call Trace: [ 459.472895] [ 459.473012] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 459.473255] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 459.473496] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 459.473762] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 459.474004] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 459.474245] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 459.474478] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 459.474766] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 459.474999] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 459.475263] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 459.475503] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 459.475731] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 459.476011] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 459.476278] [ 459.476395] [ 459.476511] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 459.476796] ? queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 459.477057] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 459.477267] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 459.477475] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 459.477742] ? up_read (kernel/locking/rwsem.c:1622) [ 459.477920] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 459.478160] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 459.478424] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 459.478679] __tcf_action_put (net/sched/act_api.c:377) [ 459.478892] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 459.479131] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 459.479338] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 459.479619] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 459.479874] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 459.480120] tcf_proto_destroy (net/sched/cls_api.c:418) [ 459.480331] tcf_chain_flush (net/sched/cls_api.c:737) [ 459.480537] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 459.480747] tcf_block_put (net/sched/cls_api.c:1543) [ 459.480943] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 459.481181] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 459.481391] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 459.481586] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 459.481881] default_device_exit_batch (net/core/dev.c:11629) [ 459.482137] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 459.482333] ? process_one_work (kernel/workqueue.c:2606) [ 459.482556] process_one_work (kernel/workqueue.c:2633) [ 459.482773] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 459.482986] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 459.483227] kthread (kernel/kthread.c:388) [ 459.483402] ? __pfx_kthread (kernel/kthread.c:341) [ 459.483603] ret_from_fork (arch/x86/kernel/process.c:147) [ 459.483799] ? __pfx_kthread (kernel/kthread.c:341) [ 459.484000] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 472.125888] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 428717 jiffies s: 6353 root: 0x8/. | [ 472.126499] rcu: blocking rcu_node structures (internal RCU debug): | [ 472.126832] Sending NMI from CPU 1 to CPUs 3: | [ 472.127084] NMI backtrace for cpu 3 [ 472.127092] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 472.127094] Workqueue: netns cleanup_net [ 472.127103] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 472.127110] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 472.127111] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 472.127113] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 472.127114] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 472.127116] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 472.127117] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 472.127117] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 472.127120] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 472.127121] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 472.127122] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 472.127123] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 472.127124] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 472.127125] PKRU: 55555554 [ 472.127126] Call Trace: [ 472.127129] [ 472.127131] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 472.127136] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 472.127141] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 472.127144] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 472.127148] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 472.127149] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 472.127154] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 472.127156] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 472.127159] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 472.127161] [ 472.127161] [ 472.127162] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 472.127168] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 472.127172] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 472.127178] ? up_read (kernel/locking/rwsem.c:1622) [ 472.127181] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 472.127186] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 472.127188] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 472.127190] __tcf_action_put (net/sched/act_api.c:377) [ 472.127197] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 472.127199] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 472.127202] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 472.127209] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 472.127214] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 472.127219] tcf_proto_destroy (net/sched/cls_api.c:418) [ 472.127222] tcf_chain_flush (net/sched/cls_api.c:737) [ 472.127225] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 472.127228] tcf_block_put (net/sched/cls_api.c:1543) [ 472.127231] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 472.127236] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 472.127240] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 472.127243] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 472.127247] default_device_exit_batch (net/core/dev.c:11629) [ 472.127250] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 472.127252] ? process_one_work (kernel/workqueue.c:2606) [ 472.127257] process_one_work (kernel/workqueue.c:2633) [ 472.127259] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 472.127262] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 472.127264] kthread (kernel/kthread.c:388) [ 472.127267] ? __pfx_kthread (kernel/kthread.c:341) [ 472.127269] ret_from_fork (arch/x86/kernel/process.c:147) [ 472.127273] ? __pfx_kthread (kernel/kthread.c:341) [ 472.127274] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 472.127279] | [ 537.482866] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 537.483211] rcu: 3-....: (493924 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=122357 | [ 537.483732] rcu: (t=494108 jiffies g=31081 q=2283 ncpus=4) [ 537.484478] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 537.485070] Workqueue: netns cleanup_net [ 537.485287] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 537.485583] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 537.486542] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 537.486819] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 537.487192] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 537.487565] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 537.487938] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 537.488311] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 537.488684] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 537.489109] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 537.489412] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 537.489786] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 537.490160] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 537.490533] PKRU: 55555554 [ 537.490680] Call Trace: [ 537.490816] [ 537.490934] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 537.491173] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 537.491423] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 537.491688] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 537.491927] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 537.492168] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 537.492401] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 537.492683] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 537.492901] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 537.493156] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 537.493395] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 537.493624] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 537.493905] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 537.494173] [ 537.494295] [ 537.494412] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 537.494695] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 537.494956] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 537.495166] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 537.495366] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 537.495633] ? up_read (kernel/locking/rwsem.c:1622) [ 537.495813] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 537.496053] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 537.496317] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 537.496571] __tcf_action_put (net/sched/act_api.c:377) [ 537.496783] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 537.496998] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 537.497204] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 537.497486] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 537.497740] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 537.497988] tcf_proto_destroy (net/sched/cls_api.c:418) [ 537.498199] tcf_chain_flush (net/sched/cls_api.c:737) [ 537.498407] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 537.498619] tcf_block_put (net/sched/cls_api.c:1543) [ 537.498815] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 537.499054] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 537.499264] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 537.499459] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 537.499745] default_device_exit_batch (net/core/dev.c:11629) [ 537.500003] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 537.500198] ? process_one_work (kernel/workqueue.c:2606) [ 537.500422] process_one_work (kernel/workqueue.c:2633) [ 537.500641] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 537.500844] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 537.501073] kthread (kernel/kthread.c:388) [ 537.501257] ? __pfx_kthread (kernel/kthread.c:341) [ 537.501458] ret_from_fork (arch/x86/kernel/process.c:147) [ 537.501655] ? __pfx_kthread (kernel/kthread.c:341) [ 537.501856] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 570.429883] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 527021 jiffies s: 6353 root: 0x8/. | [ 570.430526] rcu: blocking rcu_node structures (internal RCU debug): | [ 570.430880] Sending NMI from CPU 1 to CPUs 3: | [ 570.431149] NMI backtrace for cpu 3 [ 570.431163] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 570.431167] Workqueue: netns cleanup_net [ 570.431179] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 570.431187] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 570.431189] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 570.431190] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 570.431192] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 570.431193] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 570.431194] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 570.431195] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 570.431198] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 570.431199] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 570.431200] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 570.431201] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 570.431202] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 570.431202] PKRU: 55555554 [ 570.431203] Call Trace: [ 570.431208] [ 570.431218] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 570.431223] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 570.431227] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 570.431230] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 570.431234] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 570.431236] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 570.431241] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 570.431243] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 570.431246] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 570.431248] [ 570.431248] [ 570.431249] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 570.431255] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 570.431259] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 570.431265] ? up_read (kernel/locking/rwsem.c:1622) [ 570.431268] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 570.431273] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 570.431275] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 570.431277] __tcf_action_put (net/sched/act_api.c:377) [ 570.431287] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 570.431289] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 570.431292] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 570.431300] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 570.431305] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 570.431310] tcf_proto_destroy (net/sched/cls_api.c:418) [ 570.431313] tcf_chain_flush (net/sched/cls_api.c:737) [ 570.431316] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 570.431319] tcf_block_put (net/sched/cls_api.c:1543) [ 570.431323] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 570.431328] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 570.431332] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 570.431335] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 570.431340] default_device_exit_batch (net/core/dev.c:11629) [ 570.431342] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 570.431345] ? process_one_work (kernel/workqueue.c:2606) [ 570.431350] process_one_work (kernel/workqueue.c:2633) [ 570.431352] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 570.431355] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 570.431356] kthread (kernel/kthread.c:388) [ 570.431360] ? __pfx_kthread (kernel/kthread.c:341) [ 570.431361] ret_from_fork (arch/x86/kernel/process.c:147) [ 570.431365] ? __pfx_kthread (kernel/kthread.c:341) [ 570.431366] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 570.431371] | [ 615.500865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 615.501214] rcu: 3-....: (571909 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=141787 | [ 615.501736] rcu: (t=572126 jiffies g=31081 q=2284 ncpus=4) [ 615.502489] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 615.503083] Workqueue: netns cleanup_net [ 615.503301] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 615.503598] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 615.504554] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 615.504831] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 615.505205] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 615.505579] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 615.505955] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 615.506372] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 615.506746] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 615.507172] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 615.507478] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 615.507853] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 615.508228] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 615.508601] PKRU: 55555554 [ 615.508747] Call Trace: [ 615.508885] [ 615.509002] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 615.509242] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 615.509485] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 615.509749] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 615.509990] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 615.510232] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 615.510465] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 615.510749] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 615.510977] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 615.511231] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 615.511471] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 615.511700] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 615.511982] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 615.512249] [ 615.512366] [ 615.512483] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 615.512768] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 615.513028] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 615.513238] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 615.513438] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 615.513704] ? up_read (kernel/locking/rwsem.c:1622) [ 615.513884] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 615.514129] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 615.514392] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 615.514648] __tcf_action_put (net/sched/act_api.c:377) [ 615.514861] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 615.515077] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 615.515285] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 615.515565] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 615.515820] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 615.516068] tcf_proto_destroy (net/sched/cls_api.c:418) [ 615.516280] tcf_chain_flush (net/sched/cls_api.c:737) [ 615.516489] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 615.516698] tcf_block_put (net/sched/cls_api.c:1543) [ 615.516894] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 615.517133] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 615.517345] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 615.517540] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 615.517827] default_device_exit_batch (net/core/dev.c:11629) [ 615.518085] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 615.518281] ? process_one_work (kernel/workqueue.c:2606) [ 615.518505] process_one_work (kernel/workqueue.c:2633) [ 615.518723] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 615.518928] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 615.519156] kthread (kernel/kthread.c:388) [ 615.519331] ? __pfx_kthread (kernel/kthread.c:341) [ 615.519533] ret_from_fork (arch/x86/kernel/process.c:147) [ 615.519729] ? __pfx_kthread (kernel/kthread.c:341) [ 615.519932] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 668.733888] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 625325 jiffies s: 6353 root: 0x8/. | [ 668.734530] rcu: blocking rcu_node structures (internal RCU debug): | [ 668.734895] Sending NMI from CPU 1 to CPUs 3: | [ 668.735149] NMI backtrace for cpu 3 [ 668.735155] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 668.735157] Workqueue: netns cleanup_net [ 668.735165] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 668.735173] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 668.735174] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 668.735176] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 668.735177] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 668.735178] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 668.735179] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 668.735180] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 668.735183] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 668.735184] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 668.735185] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 668.735186] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 668.735187] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 668.735187] PKRU: 55555554 [ 668.735188] Call Trace: [ 668.735191] [ 668.735194] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 668.735199] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 668.735205] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 668.735207] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 668.735211] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 668.735213] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 668.735218] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 668.735220] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 668.735222] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 668.735225] [ 668.735225] [ 668.735226] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 668.735232] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 668.735236] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 668.735241] ? up_read (kernel/locking/rwsem.c:1622) [ 668.735245] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 668.735250] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 668.735252] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 668.735254] __tcf_action_put (net/sched/act_api.c:377) [ 668.735261] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 668.735263] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 668.735266] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 668.735273] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 668.735279] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 668.735284] tcf_proto_destroy (net/sched/cls_api.c:418) [ 668.735287] tcf_chain_flush (net/sched/cls_api.c:737) [ 668.735289] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 668.735293] tcf_block_put (net/sched/cls_api.c:1543) [ 668.735296] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 668.735300] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 668.735305] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 668.735308] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 668.735312] default_device_exit_batch (net/core/dev.c:11629) [ 668.735315] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 668.735318] ? process_one_work (kernel/workqueue.c:2606) [ 668.735323] process_one_work (kernel/workqueue.c:2633) [ 668.735325] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 668.735328] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 668.735329] kthread (kernel/kthread.c:388) [ 668.735334] ? __pfx_kthread (kernel/kthread.c:341) [ 668.735335] ret_from_fork (arch/x86/kernel/process.c:147) [ 668.735339] ? __pfx_kthread (kernel/kthread.c:341) [ 668.735340] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 668.735345] | [ 693.518865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 693.519226] rcu: 3-....: (649894 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=161254 | [ 693.519749] rcu: (t=650144 jiffies g=31081 q=2287 ncpus=4) [ 693.520507] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 693.521108] Workqueue: netns cleanup_net [ 693.521327] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 693.521638] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 693.522598] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 693.522878] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 693.523253] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 693.523630] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 693.524012] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 693.524387] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 693.524764] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 693.525191] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 693.525509] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 693.525886] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 693.526262] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 693.526639] PKRU: 55555554 [ 693.526787] Call Trace: [ 693.526925] [ 693.527043] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 693.527285] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 693.527528] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 693.527794] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 693.528039] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 693.528281] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 693.528515] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 693.528822] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 693.529078] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 693.529447] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 693.529754] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 693.529984] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 693.530265] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 693.530560] [ 693.530678] [ 693.530796] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 693.531084] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 693.531347] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 693.531563] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 693.531765] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 693.532036] ? up_read (kernel/locking/rwsem.c:1622) [ 693.532215] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 693.532463] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 693.532729] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 693.532986] __tcf_action_put (net/sched/act_api.c:377) [ 693.533199] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 693.533421] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 693.533628] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 693.533909] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 693.534165] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 693.534421] tcf_proto_destroy (net/sched/cls_api.c:418) [ 693.534633] tcf_chain_flush (net/sched/cls_api.c:737) [ 693.534841] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 693.535050] tcf_block_put (net/sched/cls_api.c:1543) [ 693.535247] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 693.535496] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 693.535705] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 693.535900] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 693.536189] default_device_exit_batch (net/core/dev.c:11629) [ 693.536452] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 693.536650] ? process_one_work (kernel/workqueue.c:2606) [ 693.536877] process_one_work (kernel/workqueue.c:2633) [ 693.537094] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 693.537298] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 693.537532] kthread (kernel/kthread.c:388) [ 693.537712] ? __pfx_kthread (kernel/kthread.c:341) [ 693.537914] ret_from_fork (arch/x86/kernel/process.c:147) [ 693.538110] ? __pfx_kthread (kernel/kthread.c:341) [ 693.538313] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 767.037885] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 723629 jiffies s: 6353 root: 0x8/. | [ 767.038765] rcu: blocking rcu_node structures (internal RCU debug): | [ 767.039253] Sending NMI from CPU 1 to CPUs 3: | [ 767.039600] NMI backtrace for cpu 3 [ 767.039609] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 767.039611] Workqueue: netns cleanup_net [ 767.039620] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 767.039628] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 767.039629] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 767.039631] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 767.039632] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 767.039633] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 767.039635] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 767.039636] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 767.039638] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 767.039639] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 767.039640] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 767.039641] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 767.039642] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 767.039643] PKRU: 55555554 [ 767.039643] Call Trace: [ 767.039647] [ 767.039650] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 767.039655] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 767.039660] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 767.039663] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 767.039667] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 767.039669] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 767.039674] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 767.039676] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 767.039678] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 767.039680] [ 767.039680] [ 767.039682] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 767.039687] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 767.039691] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 767.039697] ? up_read (kernel/locking/rwsem.c:1622) [ 767.039700] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 767.039706] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 767.039708] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 767.039710] __tcf_action_put (net/sched/act_api.c:377) [ 767.039717] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 767.039719] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 767.039722] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 767.039730] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 767.039735] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 767.039740] tcf_proto_destroy (net/sched/cls_api.c:418) [ 767.039743] tcf_chain_flush (net/sched/cls_api.c:737) [ 767.039746] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 767.039749] tcf_block_put (net/sched/cls_api.c:1543) [ 767.039753] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 767.039757] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 767.039762] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 767.039765] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 767.039770] default_device_exit_batch (net/core/dev.c:11629) [ 767.039773] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 767.039776] ? process_one_work (kernel/workqueue.c:2606) [ 767.039780] process_one_work (kernel/workqueue.c:2633) [ 767.039783] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 767.039785] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 767.039787] kthread (kernel/kthread.c:388) [ 767.039790] ? __pfx_kthread (kernel/kthread.c:341) [ 767.039792] ret_from_fork (arch/x86/kernel/process.c:147) [ 767.039795] ? __pfx_kthread (kernel/kthread.c:341) [ 767.039796] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 767.039802] | [ 771.536865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 771.537209] rcu: 3-....: (727879 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=180685 | [ 771.537730] rcu: (t=728162 jiffies g=31081 q=2287 ncpus=4) [ 771.538477] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 771.539072] Workqueue: netns cleanup_net [ 771.539293] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 771.539609] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 771.540585] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 771.540862] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 771.541243] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 771.541616] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 771.541999] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 771.542375] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 771.542757] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 771.543188] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 771.543494] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 771.543870] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 771.544253] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 771.544632] PKRU: 55555554 [ 771.544783] Call Trace: [ 771.544923] [ 771.545045] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 771.545287] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 771.545533] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 771.545805] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 771.546049] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 771.546291] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 771.546527] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 771.546811] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 771.547035] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 771.547291] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 771.547530] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 771.547761] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 771.548039] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 771.548307] [ 771.548426] [ 771.548549] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 771.548833] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 771.549095] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 771.549305] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 771.549507] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 771.549776] ? up_read (kernel/locking/rwsem.c:1622) [ 771.549965] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 771.550209] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 771.550474] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 771.550733] __tcf_action_put (net/sched/act_api.c:377) [ 771.550945] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 771.551162] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 771.551373] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 771.551657] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 771.551914] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 771.552160] tcf_proto_destroy (net/sched/cls_api.c:418) [ 771.552384] tcf_chain_flush (net/sched/cls_api.c:737) [ 771.552592] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 771.552800] tcf_block_put (net/sched/cls_api.c:1543) [ 771.552996] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 771.553238] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 771.553455] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 771.553653] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 771.553950] default_device_exit_batch (net/core/dev.c:11629) [ 771.554208] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 771.554405] ? process_one_work (kernel/workqueue.c:2606) [ 771.554633] process_one_work (kernel/workqueue.c:2633) [ 771.554850] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 771.555059] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 771.555291] kthread (kernel/kthread.c:388) [ 771.555465] ? __pfx_kthread (kernel/kthread.c:341) [ 771.555668] ret_from_fork (arch/x86/kernel/process.c:147) [ 771.555866] ? __pfx_kthread (kernel/kthread.c:341) [ 771.556069] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 771.556284] | [ 849.554867] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 849.555356] rcu: 3-....: (805879 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=200136 | [ 849.556114] rcu: (t=806180 jiffies g=31081 q=2287 ncpus=4) [ 849.557177] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 849.558040] Workqueue: netns cleanup_net [ 849.558357] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 849.558781] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 <8b> 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: 90 nop f: 90 nop 10: 90 nop 11: f3 0f 1e fa endbr64 15: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 1a: 41 55 push %r13 1c: 41 54 push %r12 1e: 55 push %rbp 1f: 48 89 fd mov %rdi,%rbp 22: 53 push %rbx 23: 66 90 xchg %ax,%ax 25: ba 01 00 00 00 mov $0x1,%edx 2a:* 8b 45 00 mov 0x0(%rbp),%eax <-- trapping instruction 2d: 85 c0 test %eax,%eax 2f: 75 14 jne 0x45 31: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 36: 85 c0 test %eax,%eax 38: 75 f0 jne 0x2a 3a: 5b pop %rbx 3b: 5d pop %rbp 3c: 41 5c pop %r12 3e: 41 5d pop %r13 Code starting with the faulting instruction =========================================== 0: 8b 45 00 mov 0x0(%rbp),%eax 3: 85 c0 test %eax,%eax 5: 75 14 jne 0x1b 7: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) c: 85 c0 test %eax,%eax e: 75 f0 jne 0x0 10: 5b pop %rbx 11: 5d pop %rbp 12: 41 5c pop %r12 14: 41 5d pop %r13 [ 849.560211] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 849.560618] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 849.561174] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 849.561727] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 849.562268] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 849.562804] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 849.563340] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 849.563963] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 849.564394] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 849.564942] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 849.565491] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 849.566033] PKRU: 55555554 [ 849.566252] Call Trace: [ 849.566454] [ 849.566628] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 849.566983] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 849.567340] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 849.567731] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 849.568094] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 849.568457] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 849.568807] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 849.569215] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 849.569544] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 849.569919] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 849.570262] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 849.570601] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 849.571000] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 849.571387] [ 849.571564] [ 849.571741] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 849.572167] ? queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 849.572549] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 849.572859] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 849.573151] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 849.573541] ? up_read (kernel/locking/rwsem.c:1622) [ 849.573805] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 849.574154] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 849.574527] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 849.574880] __tcf_action_put (net/sched/act_api.c:377) [ 849.575179] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 849.575480] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 849.575774] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 849.576197] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 849.576584] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 849.576955] tcf_proto_destroy (net/sched/cls_api.c:418) [ 849.577259] tcf_chain_flush (net/sched/cls_api.c:737) [ 849.577541] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 849.577836] tcf_block_put (net/sched/cls_api.c:1543) [ 849.578105] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 849.578437] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 849.578730] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 849.578997] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 849.579399] default_device_exit_batch (net/core/dev.c:11629) [ 849.579775] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 849.580041] ? process_one_work (kernel/workqueue.c:2606) [ 849.580347] process_one_work (kernel/workqueue.c:2633) [ 849.580644] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 849.580931] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 849.581249] kthread (kernel/kthread.c:388) [ 849.581483] ? __pfx_kthread (kernel/kthread.c:341) [ 849.581773] ret_from_fork (arch/x86/kernel/process.c:147) [ 849.582043] ? __pfx_kthread (kernel/kthread.c:341) [ 849.582320] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 865.341879] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 821933 jiffies s: 6353 root: 0x8/. | [ 865.342474] rcu: blocking rcu_node structures (internal RCU debug): | [ 865.342842] Sending NMI from CPU 1 to CPUs 3: | [ 865.343096] NMI backtrace for cpu 3 [ 865.343104] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 865.343106] Workqueue: netns cleanup_net [ 865.343115] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 865.343123] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 865.343124] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 865.343126] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 865.343127] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 865.343128] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 865.343129] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 865.343130] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 865.343133] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 865.343134] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 865.343135] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 865.343136] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 865.343137] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 865.343138] PKRU: 55555554 [ 865.343139] Call Trace: [ 865.343142] [ 865.343146] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 865.343150] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 865.343155] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 865.343158] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 865.343162] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 865.343164] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 865.343169] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 865.343172] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 865.343174] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 865.343176] [ 865.343177] [ 865.343178] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 865.343184] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 865.343188] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 865.343193] ? up_read (kernel/locking/rwsem.c:1622) [ 865.343197] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 865.343203] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 865.343204] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 865.343207] __tcf_action_put (net/sched/act_api.c:377) [ 865.343214] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 865.343216] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 865.343218] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 865.343226] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 865.343231] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 865.343237] tcf_proto_destroy (net/sched/cls_api.c:418) [ 865.343240] tcf_chain_flush (net/sched/cls_api.c:737) [ 865.343242] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 865.343245] tcf_block_put (net/sched/cls_api.c:1543) [ 865.343249] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 865.343253] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 865.343257] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 865.343260] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 865.343265] default_device_exit_batch (net/core/dev.c:11629) [ 865.343268] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 865.343270] ? process_one_work (kernel/workqueue.c:2606) [ 865.343275] process_one_work (kernel/workqueue.c:2633) [ 865.343278] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 865.343280] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 865.343282] kthread (kernel/kthread.c:388) [ 865.343286] ? __pfx_kthread (kernel/kthread.c:341) [ 865.343287] ret_from_fork (arch/x86/kernel/process.c:147) [ 865.343292] ? __pfx_kthread (kernel/kthread.c:341) [ 865.343293] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 865.343300] | [ 927.582865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 927.583212] rcu: 3-....: (883864 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=219572 | [ 927.583735] rcu: (t=884208 jiffies g=31081 q=2287 ncpus=4) [ 927.584479] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 927.585072] Workqueue: netns cleanup_net [ 927.585292] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 927.585588] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 927.586550] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 927.586826] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 927.587200] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 927.587572] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 927.587946] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 927.588319] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 927.588693] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 927.589130] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 927.589433] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 927.589809] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 927.590181] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 927.590554] PKRU: 55555554 [ 927.590703] Call Trace: [ 927.590845] [ 927.590961] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 927.591202] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 927.591447] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 927.591711] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 927.591954] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 927.592193] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 927.592429] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 927.592711] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 927.592932] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 927.593187] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 927.593425] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 927.593657] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 927.593935] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 927.594202] [ 927.594320] [ 927.594439] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 927.594722] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 927.594982] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 927.595191] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 927.595391] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 927.595659] ? up_read (kernel/locking/rwsem.c:1622) [ 927.595837] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 927.596079] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 927.596343] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 927.596600] __tcf_action_put (net/sched/act_api.c:377) [ 927.596810] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 927.597024] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 927.597234] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 927.597516] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 927.597771] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 927.598016] tcf_proto_destroy (net/sched/cls_api.c:418) [ 927.598230] tcf_chain_flush (net/sched/cls_api.c:737) [ 927.598437] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 927.598644] tcf_block_put (net/sched/cls_api.c:1543) [ 927.598840] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 927.599087] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 927.599296] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 927.599491] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 927.599777] default_device_exit_batch (net/core/dev.c:11629) [ 927.600034] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 927.600230] ? process_one_work (kernel/workqueue.c:2606) [ 927.600457] process_one_work (kernel/workqueue.c:2633) [ 927.600673] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 927.600876] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 927.601106] kthread (kernel/kthread.c:388) [ 927.601279] ? __pfx_kthread (kernel/kthread.c:341) [ 927.601483] ret_from_fork (arch/x86/kernel/process.c:147) [ 927.601679] ? __pfx_kthread (kernel/kthread.c:341) [ 927.601882] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 963.645878] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 920237 jiffies s: 6353 root: 0x8/. | [ 963.646528] rcu: blocking rcu_node structures (internal RCU debug): | [ 963.646881] Sending NMI from CPU 1 to CPUs 3: | [ 963.647144] NMI backtrace for cpu 3 [ 963.647152] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 963.647154] Workqueue: netns cleanup_net [ 963.647163] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 963.647171] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 963.647173] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 963.647174] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 963.647176] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 963.647177] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 963.647178] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 963.647179] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 963.647182] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 963.647183] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 963.647183] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 963.647184] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 963.647185] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 963.647186] PKRU: 55555554 [ 963.647186] Call Trace: [ 963.647189] [ 963.647192] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 963.647196] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 963.647202] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 963.647204] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 963.647209] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 963.647211] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 963.647215] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 963.647217] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 963.647219] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 963.647222] [ 963.647222] [ 963.647223] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 963.647229] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 963.647233] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 963.647238] ? up_read (kernel/locking/rwsem.c:1622) [ 963.647242] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 963.647248] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 963.647250] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 963.647252] __tcf_action_put (net/sched/act_api.c:377) [ 963.647259] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 963.647261] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 963.647264] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 963.647270] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 963.647275] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 963.647280] tcf_proto_destroy (net/sched/cls_api.c:418) [ 963.647283] tcf_chain_flush (net/sched/cls_api.c:737) [ 963.647286] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 963.647289] tcf_block_put (net/sched/cls_api.c:1543) [ 963.647292] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 963.647297] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 963.647301] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 963.647304] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 963.647309] default_device_exit_batch (net/core/dev.c:11629) [ 963.647312] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 963.647315] ? process_one_work (kernel/workqueue.c:2606) [ 963.647318] process_one_work (kernel/workqueue.c:2633) [ 963.647321] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 963.647323] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 963.647325] kthread (kernel/kthread.c:388) [ 963.647327] ? __pfx_kthread (kernel/kthread.c:341) [ 963.647329] ret_from_fork (arch/x86/kernel/process.c:147) [ 963.647332] ? __pfx_kthread (kernel/kthread.c:341) [ 963.647333] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 963.647340] | [ 1005.600865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1005.601215] rcu: 3-....: (961849 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=238961 | [ 1005.601743] rcu: (t=962226 jiffies g=31081 q=2288 ncpus=4) [ 1005.602490] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1005.603092] Workqueue: netns cleanup_net [ 1005.603311] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1005.603610] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1005.604578] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1005.604863] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1005.605239] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1005.605613] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1005.605994] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1005.606368] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1005.606751] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1005.607176] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1005.607481] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1005.607866] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1005.608240] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1005.608617] PKRU: 55555554 [ 1005.608786] Call Trace: [ 1005.608925] [ 1005.609040] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1005.609282] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1005.609527] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 1005.609799] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 1005.610039] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1005.610281] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1005.610517] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1005.610806] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1005.611024] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1005.611281] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1005.611521] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1005.611764] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1005.612043] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1005.612309] [ 1005.612427] [ 1005.612546] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1005.612856] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1005.613117] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1005.613327] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1005.613528] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1005.613803] ? up_read (kernel/locking/rwsem.c:1622) [ 1005.613982] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1005.614225] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1005.614490] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1005.614752] __tcf_action_put (net/sched/act_api.c:377) [ 1005.614963] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1005.615179] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1005.615388] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1005.615686] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1005.615953] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1005.616200] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1005.616413] tcf_chain_flush (net/sched/cls_api.c:737) [ 1005.616622] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1005.616840] tcf_block_put (net/sched/cls_api.c:1543) [ 1005.617037] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1005.617279] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1005.617490] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1005.617693] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1005.617982] default_device_exit_batch (net/core/dev.c:11629) [ 1005.618239] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1005.618436] ? process_one_work (kernel/workqueue.c:2606) [ 1005.618674] process_one_work (kernel/workqueue.c:2633) [ 1005.618892] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1005.619100] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1005.619333] kthread (kernel/kthread.c:388) [ 1005.619506] ? __pfx_kthread (kernel/kthread.c:341) [ 1005.619708] ret_from_fork (arch/x86/kernel/process.c:147) [ 1005.619906] ? __pfx_kthread (kernel/kthread.c:341) [ 1005.620110] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1061.949882] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 1018541 jiffies s: 6353 root: 0x8/. | [ 1061.950520] rcu: blocking rcu_node structures (internal RCU debug): | [ 1061.950900] Sending NMI from CPU 1 to CPUs 3: | [ 1061.951144] NMI backtrace for cpu 3 [ 1061.951153] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1061.951155] Workqueue: netns cleanup_net [ 1061.951164] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1061.951172] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1061.951173] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1061.951175] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1061.951176] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1061.951177] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1061.951178] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1061.951179] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1061.951181] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1061.951182] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1061.951183] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1061.951184] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1061.951185] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1061.951185] PKRU: 55555554 [ 1061.951186] Call Trace: [ 1061.951189] [ 1061.951191] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 1061.951196] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 1061.951201] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 1061.951204] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 1061.951209] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 1061.951211] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 1061.951215] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1061.951217] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1061.951220] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1061.951221] [ 1061.951222] [ 1061.951223] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1061.951229] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1061.951235] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1061.951240] ? up_read (kernel/locking/rwsem.c:1622) [ 1061.951244] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1061.951249] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1061.951250] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1061.951253] __tcf_action_put (net/sched/act_api.c:377) [ 1061.951260] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1061.951262] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1061.951265] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1061.951272] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1061.951277] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1061.951283] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1061.951285] tcf_chain_flush (net/sched/cls_api.c:737) [ 1061.951288] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1061.951291] tcf_block_put (net/sched/cls_api.c:1543) [ 1061.951295] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1061.951300] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1061.951304] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1061.951307] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1061.951311] default_device_exit_batch (net/core/dev.c:11629) [ 1061.951314] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1061.951317] ? process_one_work (kernel/workqueue.c:2606) [ 1061.951321] process_one_work (kernel/workqueue.c:2633) [ 1061.951324] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1061.951327] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1061.951328] kthread (kernel/kthread.c:388) [ 1061.951331] ? __pfx_kthread (kernel/kthread.c:341) [ 1061.951333] ret_from_fork (arch/x86/kernel/process.c:147) [ 1061.951336] ? __pfx_kthread (kernel/kthread.c:341) [ 1061.951338] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1061.951343] | [ 1083.618865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1083.619210] rcu: 3-....: (1039834 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=258278 | [ 1083.619738] rcu: (t=1040244 jiffies g=31081 q=2288 ncpus=4) [ 1083.620494] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1083.621092] Workqueue: netns cleanup_net [ 1083.621311] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1083.621605] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1083.622573] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1083.622851] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1083.623229] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1083.623605] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1083.623981] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1083.624357] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1083.624734] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1083.625156] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1083.625465] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1083.625840] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1083.626217] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1083.626590] PKRU: 55555554 [ 1083.626740] Call Trace: [ 1083.626878] [ 1083.626996] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1083.627234] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1083.627481] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 1083.627745] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 1083.627989] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1083.628240] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1083.628476] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1083.628760] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1083.628980] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1083.629238] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1083.629476] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1083.629708] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1083.629987] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1083.630255] [ 1083.630374] [ 1083.630493] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1083.630774] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1083.631037] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1083.631250] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1083.631451] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1083.631720] ? up_read (kernel/locking/rwsem.c:1622) [ 1083.631900] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1083.632143] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1083.632408] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1083.632665] __tcf_action_put (net/sched/act_api.c:377) [ 1083.632878] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1083.633096] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1083.633305] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1083.633588] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1083.633844] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1083.634092] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1083.634306] tcf_chain_flush (net/sched/cls_api.c:737) [ 1083.634513] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1083.634721] tcf_block_put (net/sched/cls_api.c:1543) [ 1083.634917] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1083.635160] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1083.635368] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1083.635562] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1083.635852] default_device_exit_batch (net/core/dev.c:11629) [ 1083.636110] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1083.636306] ? process_one_work (kernel/workqueue.c:2606) [ 1083.636534] process_one_work (kernel/workqueue.c:2633) [ 1083.636750] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1083.636953] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1083.637186] kthread (kernel/kthread.c:388) [ 1083.637361] ? __pfx_kthread (kernel/kthread.c:341) [ 1083.637564] ret_from_fork (arch/x86/kernel/process.c:147) [ 1083.637762] ? __pfx_kthread (kernel/kthread.c:341) [ 1083.637964] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1160.253881] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 1116845 jiffies s: 6353 root: 0x8/. | [ 1160.254506] rcu: blocking rcu_node structures (internal RCU debug): | [ 1160.254873] Sending NMI from CPU 1 to CPUs 3: | [ 1160.255114] NMI backtrace for cpu 3 [ 1160.255121] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1160.255123] Workqueue: netns cleanup_net [ 1160.255132] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1160.255139] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1160.255141] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1160.255143] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1160.255144] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1160.255145] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1160.255146] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1160.255147] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1160.255150] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1160.255151] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1160.255152] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1160.255153] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1160.255154] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1160.255154] PKRU: 55555554 [ 1160.255155] Call Trace: [ 1160.255158] [ 1160.255160] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 1160.255165] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 1160.255170] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 1160.255173] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 1160.255177] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 1160.255179] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 1160.255184] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1160.255186] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1160.255188] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1160.255190] [ 1160.255190] [ 1160.255191] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1160.255197] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1160.255201] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1160.255206] ? up_read (kernel/locking/rwsem.c:1622) [ 1160.255210] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1160.255216] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1160.255218] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1160.255220] __tcf_action_put (net/sched/act_api.c:377) [ 1160.255228] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1160.255230] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1160.255233] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1160.255240] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1160.255246] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1160.255251] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1160.255254] tcf_chain_flush (net/sched/cls_api.c:737) [ 1160.255257] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1160.255260] tcf_block_put (net/sched/cls_api.c:1543) [ 1160.255264] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1160.255268] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1160.255272] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1160.255275] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1160.255279] default_device_exit_batch (net/core/dev.c:11629) [ 1160.255282] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1160.255285] ? process_one_work (kernel/workqueue.c:2606) [ 1160.255290] process_one_work (kernel/workqueue.c:2633) [ 1160.255292] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1160.255295] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1160.255296] kthread (kernel/kthread.c:388) [ 1160.255299] ? __pfx_kthread (kernel/kthread.c:341) [ 1160.255300] ret_from_fork (arch/x86/kernel/process.c:147) [ 1160.255304] ? __pfx_kthread (kernel/kthread.c:341) [ 1160.255305] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1160.255312] | [ 1161.636865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1161.637192] rcu: 3-....: (1117819 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=277677 | [ 1161.637719] rcu: (t=1118262 jiffies g=31081 q=2288 ncpus=4) [ 1161.638479] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1161.639073] Workqueue: netns cleanup_net [ 1161.639289] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1161.639582] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1161.640539] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1161.640817] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1161.641190] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1161.641565] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1161.641940] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1161.642316] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1161.642688] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1161.643113] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1161.643418] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1161.643795] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1161.644171] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1161.644545] PKRU: 55555554 [ 1161.644692] Call Trace: [ 1161.644829] [ 1161.644947] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1161.645186] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1161.645431] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 1161.645695] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 1161.645936] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1161.646178] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1161.646413] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1161.646696] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1161.646915] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1161.647170] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1161.647410] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1161.647642] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1161.647930] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1161.648208] [ 1161.648327] [ 1161.648445] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1161.648728] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1161.648991] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1161.649199] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1161.649398] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1161.649666] ? up_read (kernel/locking/rwsem.c:1622) [ 1161.649845] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1161.650088] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1161.650352] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1161.650608] __tcf_action_put (net/sched/act_api.c:377) [ 1161.650818] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1161.651034] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1161.651239] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1161.651523] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1161.651777] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1161.652023] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1161.652236] tcf_chain_flush (net/sched/cls_api.c:737) [ 1161.652444] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1161.652652] tcf_block_put (net/sched/cls_api.c:1543) [ 1161.652848] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1161.653090] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1161.653298] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1161.653494] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1161.653780] default_device_exit_batch (net/core/dev.c:11629) [ 1161.654038] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1161.654234] ? process_one_work (kernel/workqueue.c:2606) [ 1161.654459] process_one_work (kernel/workqueue.c:2633) [ 1161.654676] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1161.654880] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1161.655110] kthread (kernel/kthread.c:388) [ 1161.655283] ? __pfx_kthread (kernel/kthread.c:341) [ 1161.655485] ret_from_fork (arch/x86/kernel/process.c:147) [ 1161.655681] ? __pfx_kthread (kernel/kthread.c:341) [ 1161.655883] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1161.656101] | [ 1239.654865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1239.655210] rcu: 3-....: (1195819 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=296976 | [ 1239.655738] rcu: (t=1196280 jiffies g=31081 q=2289 ncpus=4) [ 1239.656492] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1239.657087] Workqueue: netns cleanup_net [ 1239.657304] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1239.657607] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1239.658568] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1239.658845] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1239.659220] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1239.659595] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1239.659971] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1239.660343] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1239.660718] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1239.661143] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1239.661446] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1239.661821] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1239.662195] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1239.662569] PKRU: 55555554 [ 1239.662717] Call Trace: [ 1239.662854] [ 1239.662971] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1239.663212] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1239.663454] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 1239.663721] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 1239.663962] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1239.664204] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1239.664438] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1239.664723] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1239.664941] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1239.665197] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1239.665437] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1239.665667] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1239.665947] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1239.666216] [ 1239.666333] [ 1239.666450] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1239.666737] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1239.667000] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1239.667210] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1239.667417] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1239.667685] ? up_read (kernel/locking/rwsem.c:1622) [ 1239.667865] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1239.668107] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1239.668373] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1239.668631] __tcf_action_put (net/sched/act_api.c:377) [ 1239.668844] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1239.669060] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1239.669267] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1239.669549] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1239.669806] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1239.670054] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1239.670266] tcf_chain_flush (net/sched/cls_api.c:737) [ 1239.670475] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1239.670684] tcf_block_put (net/sched/cls_api.c:1543) [ 1239.670881] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1239.671121] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1239.671332] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1239.671527] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1239.671816] default_device_exit_batch (net/core/dev.c:11629) [ 1239.672073] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1239.672269] ? process_one_work (kernel/workqueue.c:2606) [ 1239.672495] process_one_work (kernel/workqueue.c:2633) [ 1239.672712] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1239.672916] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1239.673145] kthread (kernel/kthread.c:388) [ 1239.673320] ? __pfx_kthread (kernel/kthread.c:341) [ 1239.673521] ret_from_fork (arch/x86/kernel/process.c:147) [ 1239.673718] ? __pfx_kthread (kernel/kthread.c:341) [ 1239.673920] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1258.557884] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 1215149 jiffies s: 6353 root: 0x8/. | [ 1258.558760] rcu: blocking rcu_node structures (internal RCU debug): | [ 1258.559245] Sending NMI from CPU 1 to CPUs 3: | [ 1258.559591] NMI backtrace for cpu 3 [ 1258.559597] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1258.559599] Workqueue: netns cleanup_net [ 1258.559607] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1258.559615] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1258.559616] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1258.559618] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1258.559619] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1258.559620] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1258.559621] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1258.559622] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1258.559625] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1258.559626] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1258.559627] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1258.559628] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1258.559629] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1258.559630] PKRU: 55555554 [ 1258.559631] Call Trace: [ 1258.559634] [ 1258.559636] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 1258.559641] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 1258.559646] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 1258.559649] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 1258.559654] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 1258.559656] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 1258.559661] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1258.559664] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1258.559666] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1258.559668] [ 1258.559668] [ 1258.559669] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1258.559675] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1258.559679] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1258.559684] ? up_read (kernel/locking/rwsem.c:1622) [ 1258.559687] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1258.559692] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1258.559694] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1258.559696] __tcf_action_put (net/sched/act_api.c:377) [ 1258.559703] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1258.559705] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1258.559707] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1258.559714] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1258.559720] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1258.559725] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1258.559727] tcf_chain_flush (net/sched/cls_api.c:737) [ 1258.559730] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1258.559733] tcf_block_put (net/sched/cls_api.c:1543) [ 1258.559737] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1258.559741] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1258.559746] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1258.559750] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1258.559754] default_device_exit_batch (net/core/dev.c:11629) [ 1258.559757] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1258.559760] ? process_one_work (kernel/workqueue.c:2606) [ 1258.559764] process_one_work (kernel/workqueue.c:2633) [ 1258.559767] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1258.559769] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1258.559771] kthread (kernel/kthread.c:388) [ 1258.559774] ? __pfx_kthread (kernel/kthread.c:341) [ 1258.559775] ret_from_fork (arch/x86/kernel/process.c:147) [ 1258.559779] ? __pfx_kthread (kernel/kthread.c:341) [ 1258.559780] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1258.559786] | [ 1317.672865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1317.673210] rcu: 3-....: (1273804 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=316364 | [ 1317.673737] rcu: (t=1274298 jiffies g=31081 q=2289 ncpus=4) [ 1317.674487] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1317.675081] Workqueue: netns cleanup_net [ 1317.675300] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1317.675596] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1317.676553] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1317.676831] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1317.677213] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1317.677585] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1317.677959] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1317.678332] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1317.678706] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1317.679127] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1317.679430] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1317.679803] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1317.680176] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1317.680554] PKRU: 55555554 [ 1317.680701] Call Trace: [ 1317.680839] [ 1317.680955] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1317.681195] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1317.681440] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 1317.681703] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 1317.681944] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1317.682185] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1317.682418] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1317.682700] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1317.682919] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1317.683173] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1317.683414] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1317.683644] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1317.683923] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1317.684190] [ 1317.684309] [ 1317.684427] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1317.684709] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1317.684970] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1317.685179] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1317.685379] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1317.685645] ? up_read (kernel/locking/rwsem.c:1622) [ 1317.685823] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1317.686065] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1317.686327] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1317.686583] __tcf_action_put (net/sched/act_api.c:377) [ 1317.686794] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1317.687016] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1317.687223] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1317.687507] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1317.687761] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1317.688006] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1317.688220] tcf_chain_flush (net/sched/cls_api.c:737) [ 1317.688427] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1317.688634] tcf_block_put (net/sched/cls_api.c:1543) [ 1317.688830] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1317.689073] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1317.689281] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1317.689476] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1317.689762] default_device_exit_batch (net/core/dev.c:11629) [ 1317.690019] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1317.690214] ? process_one_work (kernel/workqueue.c:2606) [ 1317.690440] process_one_work (kernel/workqueue.c:2633) [ 1317.690656] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1317.690859] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1317.691089] kthread (kernel/kthread.c:388) [ 1317.691262] ? __pfx_kthread (kernel/kthread.c:341) [ 1317.691463] ret_from_fork (arch/x86/kernel/process.c:147) [ 1317.691659] ? __pfx_kthread (kernel/kthread.c:341) [ 1317.691862] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1356.861886] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 1313453 jiffies s: 6353 root: 0x8/. | [ 1356.862523] rcu: blocking rcu_node structures (internal RCU debug): | [ 1356.862858] Sending NMI from CPU 1 to CPUs 3: | [ 1356.863108] NMI backtrace for cpu 3 [ 1356.863113] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1356.863116] Workqueue: netns cleanup_net [ 1356.863123] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1356.863128] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1356.863130] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1356.863131] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1356.863133] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1356.863134] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1356.863134] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1356.863136] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1356.863138] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1356.863140] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1356.863141] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1356.863142] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1356.863142] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1356.863143] PKRU: 55555554 [ 1356.863144] Call Trace: [ 1356.863146] [ 1356.863149] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 1356.863154] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 1356.863158] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 1356.863162] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 1356.863166] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 1356.863167] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 1356.863172] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1356.863174] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1356.863176] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1356.863178] [ 1356.863179] [ 1356.863180] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1356.863185] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1356.863189] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1356.863193] ? up_read (kernel/locking/rwsem.c:1622) [ 1356.863195] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1356.863200] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1356.863202] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1356.863204] __tcf_action_put (net/sched/act_api.c:377) [ 1356.863209] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1356.863211] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1356.863213] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1356.863220] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1356.863226] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1356.863231] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1356.863234] tcf_chain_flush (net/sched/cls_api.c:737) [ 1356.863236] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1356.863239] tcf_block_put (net/sched/cls_api.c:1543) [ 1356.863242] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1356.863246] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1356.863250] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1356.863253] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1356.863257] default_device_exit_batch (net/core/dev.c:11629) [ 1356.863260] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1356.863263] ? process_one_work (kernel/workqueue.c:2606) [ 1356.863266] process_one_work (kernel/workqueue.c:2633) [ 1356.863269] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1356.863271] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1356.863273] kthread (kernel/kthread.c:388) [ 1356.863276] ? __pfx_kthread (kernel/kthread.c:341) [ 1356.863277] ret_from_fork (arch/x86/kernel/process.c:147) [ 1356.863281] ? __pfx_kthread (kernel/kthread.c:341) [ 1356.863283] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1356.863289] | [ 1395.690866] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1395.691202] rcu: 3-....: (1351789 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=335597 | [ 1395.691737] rcu: (t=1352316 jiffies g=31081 q=2289 ncpus=4) [ 1395.692499] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1395.693095] Workqueue: netns cleanup_net [ 1395.693311] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1395.693605] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1395.694567] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1395.694843] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1395.695220] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1395.695596] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1395.695972] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1395.696387] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1395.696775] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1395.697202] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1395.697506] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1395.697884] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1395.698260] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1395.698636] PKRU: 55555554 [ 1395.698783] Call Trace: [ 1395.698921] [ 1395.699038] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1395.699279] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1395.699523] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 1395.699789] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 1395.700034] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1395.700274] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1395.700508] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1395.700795] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1395.701015] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1395.701271] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1395.701512] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1395.701742] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1395.702022] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1395.702291] [ 1395.702409] [ 1395.702527] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1395.702812] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1395.703074] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1395.703285] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1395.703486] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1395.703752] ? up_read (kernel/locking/rwsem.c:1622) [ 1395.703932] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1395.704172] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1395.704439] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1395.704695] __tcf_action_put (net/sched/act_api.c:377) [ 1395.704908] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1395.705123] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1395.705331] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1395.705612] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1395.705869] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1395.706117] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1395.706329] tcf_chain_flush (net/sched/cls_api.c:737) [ 1395.706544] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1395.706754] tcf_block_put (net/sched/cls_api.c:1543) [ 1395.706951] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1395.707191] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1395.707401] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1395.707596] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1395.707886] default_device_exit_batch (net/core/dev.c:11629) [ 1395.708144] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1395.708340] ? process_one_work (kernel/workqueue.c:2606) [ 1395.708565] process_one_work (kernel/workqueue.c:2633) [ 1395.708782] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1395.708988] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1395.709217] kthread (kernel/kthread.c:388) [ 1395.709392] ? __pfx_kthread (kernel/kthread.c:341) [ 1395.709594] ret_from_fork (arch/x86/kernel/process.c:147) [ 1395.709791] ? __pfx_kthread (kernel/kthread.c:341) [ 1395.709995] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1455.165885] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 1411757 jiffies s: 6353 root: 0x8/. | [ 1455.166510] rcu: blocking rcu_node structures (internal RCU debug): | [ 1455.166855] Sending NMI from CPU 1 to CPUs 3: | [ 1455.167125] NMI backtrace for cpu 3 [ 1455.167132] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1455.167134] Workqueue: netns cleanup_net [ 1455.167142] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1455.167149] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1455.167151] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1455.167152] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1455.167154] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1455.167155] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1455.167156] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1455.167157] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1455.167159] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1455.167161] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1455.167162] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1455.167163] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1455.167164] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1455.167165] PKRU: 55555554 [ 1455.167165] Call Trace: [ 1455.167168] [ 1455.167171] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 1455.167176] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 1455.167180] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 1455.167183] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 1455.167188] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 1455.167190] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 1455.167195] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1455.167197] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1455.167199] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1455.167201] [ 1455.167202] [ 1455.167203] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1455.167209] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1455.167213] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1455.167218] ? up_read (kernel/locking/rwsem.c:1622) [ 1455.167222] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1455.167227] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1455.167229] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1455.167231] __tcf_action_put (net/sched/act_api.c:377) [ 1455.167238] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1455.167240] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1455.167242] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1455.167249] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1455.167255] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1455.167260] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1455.167263] tcf_chain_flush (net/sched/cls_api.c:737) [ 1455.167266] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1455.167269] tcf_block_put (net/sched/cls_api.c:1543) [ 1455.167273] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1455.167278] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1455.167282] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1455.167284] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1455.167289] default_device_exit_batch (net/core/dev.c:11629) [ 1455.167292] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1455.167295] ? process_one_work (kernel/workqueue.c:2606) [ 1455.167299] process_one_work (kernel/workqueue.c:2633) [ 1455.167302] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1455.167304] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1455.167306] kthread (kernel/kthread.c:388) [ 1455.167309] ? __pfx_kthread (kernel/kthread.c:341) [ 1455.167310] ret_from_fork (arch/x86/kernel/process.c:147) [ 1455.167314] ? __pfx_kthread (kernel/kthread.c:341) [ 1455.167315] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1455.167321] | [ 1473.708865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1473.709207] rcu: 3-....: (1429774 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=354841 | [ 1473.709732] rcu: (t=1430334 jiffies g=31081 q=2289 ncpus=4) [ 1473.710483] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1473.711074] Workqueue: netns cleanup_net [ 1473.711289] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1473.711584] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1473.712539] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1473.712817] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1473.713188] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1473.713561] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1473.713935] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1473.714308] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1473.714679] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1473.715107] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1473.715409] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1473.715780] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1473.716163] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1473.716535] PKRU: 55555554 [ 1473.716681] Call Trace: [ 1473.716817] [ 1473.716935] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1473.717174] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1473.717419] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 1473.717683] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 1473.717923] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1473.718163] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1473.718397] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1473.718679] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1473.718897] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1473.719152] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1473.719391] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1473.719620] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1473.719898] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1473.720163] [ 1473.720281] [ 1473.720399] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1473.720681] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1473.720942] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1473.721150] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1473.721349] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1473.721613] ? up_read (kernel/locking/rwsem.c:1622) [ 1473.721792] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1473.722034] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1473.722297] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1473.722552] __tcf_action_put (net/sched/act_api.c:377) [ 1473.722761] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1473.722975] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1473.723181] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1473.723463] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1473.723717] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1473.723962] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1473.724173] tcf_chain_flush (net/sched/cls_api.c:737) [ 1473.724380] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1473.724587] tcf_block_put (net/sched/cls_api.c:1543) [ 1473.724783] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1473.725023] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1473.725232] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1473.725426] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1473.725711] default_device_exit_batch (net/core/dev.c:11629) [ 1473.725967] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1473.726170] ? process_one_work (kernel/workqueue.c:2606) [ 1473.726395] process_one_work (kernel/workqueue.c:2633) [ 1473.726612] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1473.726814] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1473.727044] kthread (kernel/kthread.c:388) [ 1473.727216] ? __pfx_kthread (kernel/kthread.c:341) [ 1473.727417] ret_from_fork (arch/x86/kernel/process.c:147) [ 1473.727612] ? __pfx_kthread (kernel/kthread.c:341) [ 1473.727812] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1473.728028] | [ 1551.726865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1551.727205] rcu: 3-....: (1507774 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=374130 | [ 1551.727738] rcu: (t=1508352 jiffies g=31081 q=2289 ncpus=4) [ 1551.728487] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1551.729081] Workqueue: netns cleanup_net [ 1551.729299] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 1551.729592] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 <8b> 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: 90 nop f: 90 nop 10: 90 nop 11: f3 0f 1e fa endbr64 15: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 1a: 41 55 push %r13 1c: 41 54 push %r12 1e: 55 push %rbp 1f: 48 89 fd mov %rdi,%rbp 22: 53 push %rbx 23: 66 90 xchg %ax,%ax 25: ba 01 00 00 00 mov $0x1,%edx 2a:* 8b 45 00 mov 0x0(%rbp),%eax <-- trapping instruction 2d: 85 c0 test %eax,%eax 2f: 75 14 jne 0x45 31: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 36: 85 c0 test %eax,%eax 38: 75 f0 jne 0x2a 3a: 5b pop %rbx 3b: 5d pop %rbp 3c: 41 5c pop %r12 3e: 41 5d pop %r13 Code starting with the faulting instruction =========================================== 0: 8b 45 00 mov 0x0(%rbp),%eax 3: 85 c0 test %eax,%eax 5: 75 14 jne 0x1b 7: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) c: 85 c0 test %eax,%eax e: 75 f0 jne 0x0 10: 5b pop %rbx 11: 5d pop %rbp 12: 41 5c pop %r12 14: 41 5d pop %r13 [ 1551.730549] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1551.730826] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1551.731198] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1551.731568] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1551.731942] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1551.732314] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1551.732688] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1551.733108] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1551.733412] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1551.733786] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1551.734159] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1551.734533] PKRU: 55555554 [ 1551.734681] Call Trace: [ 1551.734818] [ 1551.734932] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1551.735171] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1551.735416] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 1551.735692] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 1551.735933] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1551.736176] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1551.736411] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1551.736693] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1551.736913] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1551.737168] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1551.737407] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1551.737639] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1551.737916] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1551.738185] [ 1551.738303] [ 1551.738423] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1551.738705] ? queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 1551.738966] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1551.739175] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1551.739375] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1551.739640] ? up_read (kernel/locking/rwsem.c:1622) [ 1551.739820] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1551.740062] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1551.740325] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1551.740581] __tcf_action_put (net/sched/act_api.c:377) [ 1551.740791] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1551.741006] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1551.741214] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1551.741497] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1551.741752] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1551.741997] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1551.742211] tcf_chain_flush (net/sched/cls_api.c:737) [ 1551.742418] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1551.742626] tcf_block_put (net/sched/cls_api.c:1543) [ 1551.742822] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1551.743064] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1551.743273] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1551.743468] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1551.743754] default_device_exit_batch (net/core/dev.c:11629) [ 1551.744011] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1551.744205] ? process_one_work (kernel/workqueue.c:2606) [ 1551.744431] process_one_work (kernel/workqueue.c:2633) [ 1551.744647] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1551.744849] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1551.745080] kthread (kernel/kthread.c:388) [ 1551.745252] ? __pfx_kthread (kernel/kthread.c:341) [ 1551.745455] ret_from_fork (arch/x86/kernel/process.c:147) [ 1551.745658] ? __pfx_kthread (kernel/kthread.c:341) [ 1551.745862] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1553.469877] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 1510061 jiffies s: 6353 root: 0x8/. | [ 1553.470481] rcu: blocking rcu_node structures (internal RCU debug): | [ 1553.470814] Sending NMI from CPU 1 to CPUs 3: | [ 1553.471093] NMI backtrace for cpu 3 [ 1553.471098] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1553.471099] Workqueue: netns cleanup_net [ 1553.471101] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1553.471103] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1553.471104] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1553.471105] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1553.471106] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1553.471107] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1553.471108] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1553.471108] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1553.471111] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1553.471112] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1553.471112] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1553.471113] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1553.471114] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1553.471114] PKRU: 55555554 [ 1553.471115] Call Trace: [ 1553.471115] [ 1553.471116] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 1553.471121] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 1553.471124] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 1553.471127] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 1553.471130] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 1553.471132] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 1553.471135] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1553.471137] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1553.471139] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1553.471141] [ 1553.471141] [ 1553.471142] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1553.471144] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1553.471147] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1553.471149] ? up_read (kernel/locking/rwsem.c:1622) [ 1553.471151] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1553.471154] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1553.471156] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1553.471157] __tcf_action_put (net/sched/act_api.c:377) [ 1553.471160] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1553.471162] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1553.471164] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1553.471169] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1553.471174] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1553.471179] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1553.471181] tcf_chain_flush (net/sched/cls_api.c:737) [ 1553.471183] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1553.471186] tcf_block_put (net/sched/cls_api.c:1543) [ 1553.471188] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1553.471192] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1553.471194] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1553.471196] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1553.471198] default_device_exit_batch (net/core/dev.c:11629) [ 1553.471200] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1553.471203] ? process_one_work (kernel/workqueue.c:2606) [ 1553.471204] process_one_work (kernel/workqueue.c:2633) [ 1553.471207] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1553.471209] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1553.471211] kthread (kernel/kthread.c:388) [ 1553.471212] ? __pfx_kthread (kernel/kthread.c:341) [ 1553.471213] ret_from_fork (arch/x86/kernel/process.c:147) [ 1553.471216] ? __pfx_kthread (kernel/kthread.c:341) [ 1553.471217] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1553.471220] | [ 1629.744866] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1629.745211] rcu: 3-....: (1585759 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=393360 | [ 1629.745745] rcu: (t=1586370 jiffies g=31081 q=2289 ncpus=4) [ 1629.746500] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1629.747095] Workqueue: netns cleanup_net [ 1629.747310] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1629.747607] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1629.748570] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1629.748853] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1629.749228] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1629.749600] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1629.749978] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1629.750355] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1629.750730] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1629.751153] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1629.751459] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1629.751834] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1629.752208] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1629.752584] PKRU: 55555554 [ 1629.752731] Call Trace: [ 1629.752870] [ 1629.752987] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1629.753228] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1629.753473] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 1629.753739] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 1629.753980] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1629.754224] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1629.754460] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1629.754744] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1629.754963] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1629.755219] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1629.755468] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1629.755699] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1629.755977] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1629.756244] [ 1629.756363] [ 1629.756482] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1629.756765] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1629.757026] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1629.757236] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1629.757436] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1629.757703] ? up_read (kernel/locking/rwsem.c:1622) [ 1629.757883] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1629.758126] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1629.758390] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1629.758648] __tcf_action_put (net/sched/act_api.c:377) [ 1629.758859] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1629.759075] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1629.759282] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1629.759565] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1629.759820] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1629.760067] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1629.760280] tcf_chain_flush (net/sched/cls_api.c:737) [ 1629.760487] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1629.760694] tcf_block_put (net/sched/cls_api.c:1543) [ 1629.760892] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1629.761134] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1629.761343] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1629.761537] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1629.761824] default_device_exit_batch (net/core/dev.c:11629) [ 1629.762082] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1629.762278] ? process_one_work (kernel/workqueue.c:2606) [ 1629.762504] process_one_work (kernel/workqueue.c:2633) [ 1629.762720] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1629.762924] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1629.763155] kthread (kernel/kthread.c:388) [ 1629.763327] ? __pfx_kthread (kernel/kthread.c:341) [ 1629.763529] ret_from_fork (arch/x86/kernel/process.c:147) [ 1629.763725] ? __pfx_kthread (kernel/kthread.c:341) [ 1629.763928] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1651.773884] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 1608365 jiffies s: 6353 root: 0x8/. | [ 1651.774490] rcu: blocking rcu_node structures (internal RCU debug): | [ 1651.774820] Sending NMI from CPU 1 to CPUs 3: | [ 1651.775071] NMI backtrace for cpu 3 [ 1651.775077] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1651.775079] Workqueue: netns cleanup_net [ 1651.775084] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1651.775089] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1651.775091] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1651.775093] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1651.775094] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1651.775095] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1651.775096] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1651.775097] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1651.775100] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1651.775111] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1651.775111] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1651.775112] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1651.775113] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1651.775114] PKRU: 55555554 [ 1651.775115] Call Trace: [ 1651.775117] [ 1651.775119] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 1651.775124] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 1651.775128] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 1651.775131] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 1651.775134] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 1651.775136] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 1651.775140] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1651.775143] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1651.775145] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1651.775147] [ 1651.775148] [ 1651.775149] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1651.775153] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1651.775156] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1651.775160] ? up_read (kernel/locking/rwsem.c:1622) [ 1651.775162] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1651.775166] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1651.775168] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1651.775170] __tcf_action_put (net/sched/act_api.c:377) [ 1651.775175] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1651.775177] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1651.775179] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1651.775185] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1651.775191] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1651.775196] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1651.775198] tcf_chain_flush (net/sched/cls_api.c:737) [ 1651.775200] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1651.775203] tcf_block_put (net/sched/cls_api.c:1543) [ 1651.775206] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1651.775210] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1651.775212] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1651.775215] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1651.775217] default_device_exit_batch (net/core/dev.c:11629) [ 1651.775220] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1651.775222] ? process_one_work (kernel/workqueue.c:2606) [ 1651.775225] process_one_work (kernel/workqueue.c:2633) [ 1651.775228] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1651.775230] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1651.775232] kthread (kernel/kthread.c:388) [ 1651.775234] ? __pfx_kthread (kernel/kthread.c:341) [ 1651.775236] ret_from_fork (arch/x86/kernel/process.c:147) [ 1651.775240] ? __pfx_kthread (kernel/kthread.c:341) [ 1651.775241] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1651.775246] | [ 1707.762866] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1707.763215] rcu: 3-....: (1663744 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=412409 | [ 1707.763750] rcu: (t=1664388 jiffies g=31081 q=2289 ncpus=4) [ 1707.764506] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1707.765121] Workqueue: netns cleanup_net [ 1707.765342] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1707.765639] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1707.766606] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1707.766891] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1707.767269] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1707.767643] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1707.768023] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1707.768399] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1707.768785] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1707.769208] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1707.769515] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1707.769901] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1707.770276] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1707.770654] PKRU: 55555554 [ 1707.770806] Call Trace: [ 1707.770946] [ 1707.771062] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1707.771307] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1707.771558] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 1707.771826] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 1707.772072] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1707.772314] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1707.772552] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1707.772851] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1707.773073] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1707.773333] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1707.773576] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1707.773812] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1707.774094] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1707.774366] [ 1707.774487] [ 1707.774608] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1707.774913] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1707.775176] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1707.775386] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1707.775588] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1707.775857] ? up_read (kernel/locking/rwsem.c:1622) [ 1707.776038] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1707.776282] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1707.776547] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1707.776806] __tcf_action_put (net/sched/act_api.c:377) [ 1707.777021] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1707.777238] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1707.777448] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1707.777732] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1707.777992] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1707.778237] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1707.778453] tcf_chain_flush (net/sched/cls_api.c:737) [ 1707.778669] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1707.778880] tcf_block_put (net/sched/cls_api.c:1543) [ 1707.779079] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1707.779321] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1707.779529] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1707.779725] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1707.780014] default_device_exit_batch (net/core/dev.c:11629) [ 1707.780271] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1707.780468] ? process_one_work (kernel/workqueue.c:2606) [ 1707.780697] process_one_work (kernel/workqueue.c:2633) [ 1707.780916] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1707.781120] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1707.781352] kthread (kernel/kthread.c:388) [ 1707.781525] ? __pfx_kthread (kernel/kthread.c:341) [ 1707.781728] ret_from_fork (arch/x86/kernel/process.c:147) [ 1707.781926] ? __pfx_kthread (kernel/kthread.c:341) [ 1707.782131] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1750.077875] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 1706669 jiffies s: 6353 root: 0x8/. | [ 1750.078481] rcu: blocking rcu_node structures (internal RCU debug): | [ 1750.078814] Sending NMI from CPU 1 to CPUs 3: | [ 1750.079085] NMI backtrace for cpu 3 [ 1750.079093] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1750.079095] Workqueue: netns cleanup_net [ 1750.079103] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1750.079110] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1750.079112] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1750.079114] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1750.079115] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1750.079116] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1750.079117] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1750.079118] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1750.079121] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1750.079122] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1750.079123] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1750.079124] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1750.079124] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1750.079125] PKRU: 55555554 [ 1750.079125] Call Trace: [ 1750.079128] [ 1750.079131] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 1750.079136] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 1750.079141] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 1750.079144] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 1750.079148] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 1750.079150] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 1750.079154] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1750.079157] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1750.079159] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1750.079161] [ 1750.079161] [ 1750.079162] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1750.079168] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1750.079172] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1750.079177] ? up_read (kernel/locking/rwsem.c:1622) [ 1750.079181] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1750.079186] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1750.079188] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1750.079190] __tcf_action_put (net/sched/act_api.c:377) [ 1750.079197] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1750.079199] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1750.079201] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1750.079208] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1750.079214] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1750.079219] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1750.079222] tcf_chain_flush (net/sched/cls_api.c:737) [ 1750.079225] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1750.079228] tcf_block_put (net/sched/cls_api.c:1543) [ 1750.079232] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1750.079237] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1750.079241] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1750.079244] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1750.079249] default_device_exit_batch (net/core/dev.c:11629) [ 1750.079251] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1750.079254] ? process_one_work (kernel/workqueue.c:2606) [ 1750.079259] process_one_work (kernel/workqueue.c:2633) [ 1750.079262] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1750.079264] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1750.079266] kthread (kernel/kthread.c:388) [ 1750.079269] ? __pfx_kthread (kernel/kthread.c:341) [ 1750.079270] ret_from_fork (arch/x86/kernel/process.c:147) [ 1750.079275] ? __pfx_kthread (kernel/kthread.c:341) [ 1750.079276] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1750.079281] | [ 1785.780867] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1785.781212] rcu: 3-....: (1741729 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=431488 | [ 1785.781738] rcu: (t=1742406 jiffies g=31081 q=2289 ncpus=4) [ 1785.782490] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1785.783085] Workqueue: netns cleanup_net [ 1785.783303] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1785.783599] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1785.784564] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1785.784841] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1785.785216] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1785.785588] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1785.785965] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1785.786339] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1785.786712] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1785.787136] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1785.787440] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1785.787817] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1785.788189] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1785.788563] PKRU: 55555554 [ 1785.788710] Call Trace: [ 1785.788849] [ 1785.788964] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1785.789206] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1785.789450] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 1785.789714] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 1785.789958] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1785.790199] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1785.790434] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1785.790715] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1785.790934] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1785.791189] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1785.791428] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1785.791660] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1785.791941] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1785.792208] [ 1785.792328] [ 1785.792447] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1785.792729] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1785.792990] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1785.793200] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1785.793401] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1785.793666] ? up_read (kernel/locking/rwsem.c:1622) [ 1785.793846] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1785.794090] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1785.794355] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1785.794620] __tcf_action_put (net/sched/act_api.c:377) [ 1785.794839] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1785.795054] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1785.795264] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1785.795548] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1785.795804] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1785.796049] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1785.796263] tcf_chain_flush (net/sched/cls_api.c:737) [ 1785.796470] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1785.796678] tcf_block_put (net/sched/cls_api.c:1543) [ 1785.796875] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1785.797118] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1785.797326] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1785.797521] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1785.797807] default_device_exit_batch (net/core/dev.c:11629) [ 1785.798064] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1785.798258] ? process_one_work (kernel/workqueue.c:2606) [ 1785.798486] process_one_work (kernel/workqueue.c:2633) [ 1785.798702] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1785.798906] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1785.799137] kthread (kernel/kthread.c:388) [ 1785.799309] ? __pfx_kthread (kernel/kthread.c:341) [ 1785.799512] ret_from_fork (arch/x86/kernel/process.c:147) [ 1785.799708] ? __pfx_kthread (kernel/kthread.c:341) [ 1785.799912] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1848.382880] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 1804974 jiffies s: 6353 root: 0x8/. | [ 1848.383712] rcu: blocking rcu_node structures (internal RCU debug): | [ 1848.384250] Sending NMI from CPU 1 to CPUs 3: | [ 1848.384582] NMI backtrace for cpu 3 [ 1848.384588] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1848.384590] Workqueue: netns cleanup_net [ 1848.384598] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 1848.384604] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 <8b> 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: 90 nop f: 90 nop 10: 90 nop 11: f3 0f 1e fa endbr64 15: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 1a: 41 55 push %r13 1c: 41 54 push %r12 1e: 55 push %rbp 1f: 48 89 fd mov %rdi,%rbp 22: 53 push %rbx 23: 66 90 xchg %ax,%ax 25: ba 01 00 00 00 mov $0x1,%edx 2a:* 8b 45 00 mov 0x0(%rbp),%eax <-- trapping instruction 2d: 85 c0 test %eax,%eax 2f: 75 14 jne 0x45 31: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 36: 85 c0 test %eax,%eax 38: 75 f0 jne 0x2a 3a: 5b pop %rbx 3b: 5d pop %rbp 3c: 41 5c pop %r12 3e: 41 5d pop %r13 Code starting with the faulting instruction =========================================== 0: 8b 45 00 mov 0x0(%rbp),%eax 3: 85 c0 test %eax,%eax 5: 75 14 jne 0x1b 7: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) c: 85 c0 test %eax,%eax e: 75 f0 jne 0x0 10: 5b pop %rbx 11: 5d pop %rbp 12: 41 5c pop %r12 14: 41 5d pop %r13 [ 1848.384606] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1848.384607] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1848.384609] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1848.384610] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1848.384611] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1848.384612] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1848.384614] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1848.384615] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1848.384616] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1848.384617] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1848.384617] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1848.384618] PKRU: 55555554 [ 1848.384619] Call Trace: [ 1848.384622] [ 1848.384625] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 1848.384629] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 1848.384634] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 1848.384636] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 1848.384641] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 1848.384642] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 1848.384647] ? queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 1848.384649] ? queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 1848.384651] ? queued_spin_lock_slowpath (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 1848.384653] [ 1848.384653] [ 1848.384654] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1848.384659] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1848.384664] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1848.384667] ? up_read (kernel/locking/rwsem.c:1622) [ 1848.384671] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1848.384675] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1848.384676] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1848.384679] __tcf_action_put (net/sched/act_api.c:377) [ 1848.384685] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1848.384687] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1848.384690] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1848.384697] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1848.384702] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1848.384708] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1848.384710] tcf_chain_flush (net/sched/cls_api.c:737) [ 1848.384713] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1848.384716] tcf_block_put (net/sched/cls_api.c:1543) [ 1848.384719] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1848.384723] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1848.384726] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1848.384729] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1848.384733] default_device_exit_batch (net/core/dev.c:11629) [ 1848.384736] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1848.384739] ? process_one_work (kernel/workqueue.c:2606) [ 1848.384743] process_one_work (kernel/workqueue.c:2633) [ 1848.384746] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1848.384748] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1848.384750] kthread (kernel/kthread.c:388) [ 1848.384753] ? __pfx_kthread (kernel/kthread.c:341) [ 1848.384754] ret_from_fork (arch/x86/kernel/process.c:147) [ 1848.384758] ? __pfx_kthread (kernel/kthread.c:341) [ 1848.384760] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1848.384764] | [ 1863.798866] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1863.799202] rcu: 3-....: (1819714 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=450589 | [ 1863.799731] rcu: (t=1820424 jiffies g=31081 q=2289 ncpus=4) [ 1863.800488] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1863.801084] Workqueue: netns cleanup_net [ 1863.801300] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1863.801594] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1863.802555] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1863.802830] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1863.803206] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1863.803581] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1863.803957] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1863.804337] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1863.804713] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1863.805136] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1863.805439] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1863.805816] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1863.806191] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1863.806565] PKRU: 55555554 [ 1863.806713] Call Trace: [ 1863.806849] [ 1863.806966] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1863.807204] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1863.807448] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 1863.807717] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 1863.807961] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1863.808201] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1863.808434] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1863.808719] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1863.808937] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1863.809193] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1863.809433] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1863.809663] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1863.809942] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1863.810208] [ 1863.810325] [ 1863.810444] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1863.810728] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1863.810990] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1863.811200] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1863.811401] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1863.811667] ? up_read (kernel/locking/rwsem.c:1622) [ 1863.811845] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1863.812085] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1863.812349] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1863.812605] __tcf_action_put (net/sched/act_api.c:377) [ 1863.812815] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1863.813032] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1863.813239] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1863.813520] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1863.813775] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1863.814029] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1863.814240] tcf_chain_flush (net/sched/cls_api.c:737) [ 1863.814447] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1863.814656] tcf_block_put (net/sched/cls_api.c:1543) [ 1863.814851] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1863.815092] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1863.815299] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1863.815494] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1863.815783] default_device_exit_batch (net/core/dev.c:11629) [ 1863.816042] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1863.816237] ? process_one_work (kernel/workqueue.c:2606) [ 1863.816462] process_one_work (kernel/workqueue.c:2633) [ 1863.816678] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1863.816881] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1863.817111] kthread (kernel/kthread.c:388) [ 1863.817285] ? __pfx_kthread (kernel/kthread.c:341) [ 1863.817486] ret_from_fork (arch/x86/kernel/process.c:147) [ 1863.817682] ? __pfx_kthread (kernel/kthread.c:341) [ 1863.817884] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1863.818098] | [ 1941.816865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 1941.817207] rcu: 3-....: (1897714 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=469603 | [ 1941.817734] rcu: (t=1898442 jiffies g=31081 q=2290 ncpus=4) [ 1941.818488] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1941.819083] Workqueue: netns cleanup_net [ 1941.819302] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1941.819599] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1941.820562] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1941.820839] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1941.821214] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1941.821593] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1941.821970] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1941.822345] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1941.822720] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1941.823143] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1941.823448] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1941.823831] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1941.824203] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1941.824578] PKRU: 55555554 [ 1941.824726] Call Trace: [ 1941.824864] [ 1941.824979] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 1941.825220] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 1941.825466] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 1941.825731] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 1941.825973] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 1941.826213] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 1941.826448] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 1941.826730] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 1941.826949] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 1941.827204] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 1941.827444] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 1941.827675] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 1941.827954] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 1941.828221] [ 1941.828340] [ 1941.828459] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 1941.828740] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1941.829002] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1941.829215] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1941.829416] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1941.829685] ? up_read (kernel/locking/rwsem.c:1622) [ 1941.829864] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1941.830105] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1941.830369] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1941.830625] __tcf_action_put (net/sched/act_api.c:377) [ 1941.830837] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1941.831052] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1941.831260] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1941.831543] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1941.831799] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1941.832048] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1941.832263] tcf_chain_flush (net/sched/cls_api.c:737) [ 1941.832471] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1941.832679] tcf_block_put (net/sched/cls_api.c:1543) [ 1941.832875] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1941.833119] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1941.833328] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1941.833526] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1941.833829] default_device_exit_batch (net/core/dev.c:11629) [ 1941.834086] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1941.834281] ? process_one_work (kernel/workqueue.c:2606) [ 1941.834508] process_one_work (kernel/workqueue.c:2633) [ 1941.834724] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1941.834928] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1941.835158] kthread (kernel/kthread.c:388) [ 1941.835331] ? __pfx_kthread (kernel/kthread.c:341) [ 1941.835533] ret_from_fork (arch/x86/kernel/process.c:147) [ 1941.835731] ? __pfx_kthread (kernel/kthread.c:341) [ 1941.835935] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1946.686879] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 1903278 jiffies s: 6353 root: 0x8/. | [ 1946.687500] rcu: blocking rcu_node structures (internal RCU debug): | [ 1946.687830] Sending NMI from CPU 1 to CPUs 3: | [ 1946.688077] NMI backtrace for cpu 3 [ 1946.688081] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 1946.688083] Workqueue: netns cleanup_net [ 1946.688087] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1946.688090] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 1946.688092] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 1946.688093] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 1946.688094] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 1946.688094] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 1946.688095] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 1946.688096] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 1946.688098] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 1946.688100] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 1946.688101] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 1946.688102] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 1946.688102] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 1946.688103] PKRU: 55555554 [ 1946.688104] Call Trace: [ 1946.688105] [ 1946.688107] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 1946.688111] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 1946.688115] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 1946.688118] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 1946.688121] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 1946.688123] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 1946.688126] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1946.688128] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1946.688130] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 1946.688132] [ 1946.688133] [ 1946.688133] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 1946.688137] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 1946.688141] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1946.688143] ? up_read (kernel/locking/rwsem.c:1622) [ 1946.688145] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 1946.688148] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1946.688150] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 1946.688152] __tcf_action_put (net/sched/act_api.c:377) [ 1946.688156] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 1946.688158] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 1946.688159] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 1946.688165] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 1946.688170] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 1946.688175] tcf_proto_destroy (net/sched/cls_api.c:418) [ 1946.688177] tcf_chain_flush (net/sched/cls_api.c:737) [ 1946.688180] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 1946.688182] tcf_block_put (net/sched/cls_api.c:1543) [ 1946.688185] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 1946.688189] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 1946.688191] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 1946.688193] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 1946.688196] default_device_exit_batch (net/core/dev.c:11629) [ 1946.688199] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 1946.688201] ? process_one_work (kernel/workqueue.c:2606) [ 1946.688204] process_one_work (kernel/workqueue.c:2633) [ 1946.688206] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 1946.688209] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 1946.688210] kthread (kernel/kthread.c:388) [ 1946.688212] ? __pfx_kthread (kernel/kthread.c:341) [ 1946.688214] ret_from_fork (arch/x86/kernel/process.c:147) [ 1946.688217] ? __pfx_kthread (kernel/kthread.c:341) [ 1946.688219] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 1946.688223] | [ 2019.834867] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2019.835209] rcu: 3-....: (1975699 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=488642 | [ 2019.835734] rcu: (t=1976460 jiffies g=31081 q=2290 ncpus=4) [ 2019.836486] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2019.837080] Workqueue: netns cleanup_net [ 2019.837296] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2019.837589] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2019.838551] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2019.838827] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2019.839203] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2019.839576] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2019.839950] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2019.840323] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2019.840699] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2019.841121] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2019.841426] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2019.841801] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2019.842177] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2019.842551] PKRU: 55555554 [ 2019.842700] Call Trace: [ 2019.842836] [ 2019.842954] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2019.843202] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2019.843447] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 2019.843711] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 2019.843955] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2019.844197] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2019.844433] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2019.844718] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2019.844938] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2019.845196] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2019.845436] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2019.845666] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2019.845944] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2019.846213] [ 2019.846331] [ 2019.846450] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2019.846733] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2019.846995] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2019.847205] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2019.847407] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2019.847674] ? up_read (kernel/locking/rwsem.c:1622) [ 2019.847853] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2019.848094] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2019.848357] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2019.848614] __tcf_action_put (net/sched/act_api.c:377) [ 2019.848827] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2019.849044] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2019.849252] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2019.849535] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2019.849791] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2019.850040] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2019.850252] tcf_chain_flush (net/sched/cls_api.c:737) [ 2019.850461] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2019.850670] tcf_block_put (net/sched/cls_api.c:1543) [ 2019.850867] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2019.851109] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2019.851317] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2019.851513] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2019.851804] default_device_exit_batch (net/core/dev.c:11629) [ 2019.852062] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2019.852259] ? process_one_work (kernel/workqueue.c:2606) [ 2019.852484] process_one_work (kernel/workqueue.c:2633) [ 2019.852699] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2019.852904] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2019.853141] kthread (kernel/kthread.c:388) [ 2019.853316] ? __pfx_kthread (kernel/kthread.c:341) [ 2019.853518] ret_from_fork (arch/x86/kernel/process.c:147) [ 2019.853714] ? __pfx_kthread (kernel/kthread.c:341) [ 2019.853917] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2044.990882] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2001582 jiffies s: 6353 root: 0x8/. | [ 2044.991653] rcu: blocking rcu_node structures (internal RCU debug): | [ 2044.992082] Sending NMI from CPU 1 to CPUs 3: | [ 2044.992393] NMI backtrace for cpu 3 [ 2044.992399] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2044.992401] Workqueue: netns cleanup_net [ 2044.992407] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2044.992412] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2044.992414] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2044.992415] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2044.992417] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2044.992418] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2044.992418] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2044.992420] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2044.992422] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2044.992423] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2044.992424] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2044.992425] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2044.992426] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2044.992427] PKRU: 55555554 [ 2044.992428] Call Trace: [ 2044.992430] [ 2044.992432] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 2044.992436] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 2044.992440] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 2044.992443] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 2044.992446] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 2044.992448] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 2044.992452] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2044.992454] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2044.992456] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2044.992458] [ 2044.992459] [ 2044.992459] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2044.992464] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2044.992468] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2044.992471] ? up_read (kernel/locking/rwsem.c:1622) [ 2044.992474] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2044.992479] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2044.992481] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2044.992483] __tcf_action_put (net/sched/act_api.c:377) [ 2044.992489] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2044.992491] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2044.992493] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2044.992500] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2044.992505] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2044.992511] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2044.992513] tcf_chain_flush (net/sched/cls_api.c:737) [ 2044.992516] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2044.992519] tcf_block_put (net/sched/cls_api.c:1543) [ 2044.992522] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2044.992526] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2044.992529] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2044.992532] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2044.992535] default_device_exit_batch (net/core/dev.c:11629) [ 2044.992538] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2044.992540] ? process_one_work (kernel/workqueue.c:2606) [ 2044.992544] process_one_work (kernel/workqueue.c:2633) [ 2044.992546] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2044.992549] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2044.992550] kthread (kernel/kthread.c:388) [ 2044.992552] ? __pfx_kthread (kernel/kthread.c:341) [ 2044.992553] ret_from_fork (arch/x86/kernel/process.c:147) [ 2044.992556] ? __pfx_kthread (kernel/kthread.c:341) [ 2044.992558] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2044.992562] | [ 2097.852865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2097.853209] rcu: 3-....: (2053684 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=507751 | [ 2097.853742] rcu: (t=2054478 jiffies g=31081 q=2290 ncpus=4) [ 2097.854501] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2097.855099] Workqueue: netns cleanup_net [ 2097.855317] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2097.855615] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2097.856587] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2097.856868] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2097.857242] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2097.857618] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2097.857995] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2097.858370] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2097.858746] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2097.859173] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2097.859480] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2097.859857] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2097.860235] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2097.860610] PKRU: 55555554 [ 2097.860758] Call Trace: [ 2097.860896] [ 2097.861014] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2097.861254] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2097.861500] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 2097.861765] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 2097.862007] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2097.862250] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2097.862486] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2097.862777] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2097.862996] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2097.863253] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2097.863495] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2097.863727] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2097.864008] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2097.864275] [ 2097.864394] [ 2097.864512] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2097.864796] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2097.865058] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2097.865267] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2097.865468] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2097.865734] ? up_read (kernel/locking/rwsem.c:1622) [ 2097.865914] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2097.866157] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2097.866422] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2097.866680] __tcf_action_put (net/sched/act_api.c:377) [ 2097.866891] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2097.867106] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2097.867313] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2097.867597] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2097.867853] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2097.868100] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2097.868314] tcf_chain_flush (net/sched/cls_api.c:737) [ 2097.868522] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2097.868731] tcf_block_put (net/sched/cls_api.c:1543) [ 2097.868927] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2097.869169] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2097.869379] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2097.869574] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2097.869861] default_device_exit_batch (net/core/dev.c:11629) [ 2097.870120] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2097.870316] ? process_one_work (kernel/workqueue.c:2606) [ 2097.870543] process_one_work (kernel/workqueue.c:2633) [ 2097.870761] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2097.870965] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2097.871196] kthread (kernel/kthread.c:388) [ 2097.871370] ? __pfx_kthread (kernel/kthread.c:341) [ 2097.871572] ret_from_fork (arch/x86/kernel/process.c:147) [ 2097.871769] ? __pfx_kthread (kernel/kthread.c:341) [ 2097.871973] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2143.293881] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2099885 jiffies s: 6353 root: 0x8/. | [ 2143.294500] rcu: blocking rcu_node structures (internal RCU debug): | [ 2143.294835] Sending NMI from CPU 1 to CPUs 3: | [ 2143.295086] NMI backtrace for cpu 3 [ 2143.295092] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2143.295094] Workqueue: netns cleanup_net [ 2143.295103] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2143.295109] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2143.295111] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2143.295112] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2143.295114] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2143.295115] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2143.295116] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2143.295117] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2143.295119] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2143.295120] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2143.295121] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2143.295122] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2143.295123] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2143.295124] PKRU: 55555554 [ 2143.295125] Call Trace: [ 2143.295127] [ 2143.295130] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 2143.295135] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 2143.295140] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 2143.295143] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 2143.295147] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 2143.295149] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 2143.295153] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2143.295156] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2143.295158] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2143.295160] [ 2143.295160] [ 2143.295161] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2143.295167] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2143.295172] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2143.295177] ? up_read (kernel/locking/rwsem.c:1622) [ 2143.295181] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2143.295186] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2143.295187] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2143.295190] __tcf_action_put (net/sched/act_api.c:377) [ 2143.295196] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2143.295198] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2143.295200] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2143.295207] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2143.295213] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2143.295218] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2143.295221] tcf_chain_flush (net/sched/cls_api.c:737) [ 2143.295223] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2143.295226] tcf_block_put (net/sched/cls_api.c:1543) [ 2143.295230] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2143.295234] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2143.295238] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2143.295241] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2143.295245] default_device_exit_batch (net/core/dev.c:11629) [ 2143.295248] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2143.295251] ? process_one_work (kernel/workqueue.c:2606) [ 2143.295255] process_one_work (kernel/workqueue.c:2633) [ 2143.295259] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2143.295261] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2143.295263] kthread (kernel/kthread.c:388) [ 2143.295266] ? __pfx_kthread (kernel/kthread.c:341) [ 2143.295267] ret_from_fork (arch/x86/kernel/process.c:147) [ 2143.295271] ? __pfx_kthread (kernel/kthread.c:341) [ 2143.295272] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2143.295277] | [ 2175.870866] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2175.871214] rcu: 3-....: (2131669 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=526846 | [ 2175.871741] rcu: (t=2132496 jiffies g=31081 q=2290 ncpus=4) [ 2175.872501] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2175.873096] Workqueue: netns cleanup_net [ 2175.873313] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2175.873613] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2175.874575] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2175.874854] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2175.875228] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2175.875601] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2175.875978] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2175.876354] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2175.876728] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2175.877155] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2175.877459] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2175.877834] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2175.878210] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2175.878585] PKRU: 55555554 [ 2175.878732] Call Trace: [ 2175.878870] [ 2175.878986] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2175.879226] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2175.879471] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 2175.879737] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 2175.879977] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2175.880221] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2175.880456] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2175.880739] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2175.880958] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2175.881215] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2175.881455] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2175.881687] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2175.881966] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2175.882234] [ 2175.882361] [ 2175.882480] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2175.882763] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2175.883025] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2175.883236] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2175.883436] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2175.883702] ? up_read (kernel/locking/rwsem.c:1622) [ 2175.883881] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2175.884124] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2175.884388] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2175.884644] __tcf_action_put (net/sched/act_api.c:377) [ 2175.884855] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2175.885071] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2175.885278] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2175.885563] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2175.885818] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2175.886066] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2175.886314] tcf_chain_flush (net/sched/cls_api.c:737) [ 2175.886523] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2175.886732] tcf_block_put (net/sched/cls_api.c:1543) [ 2175.886931] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2175.887174] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2175.887384] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2175.887579] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2175.887866] default_device_exit_batch (net/core/dev.c:11629) [ 2175.888124] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2175.888319] ? process_one_work (kernel/workqueue.c:2606) [ 2175.888546] process_one_work (kernel/workqueue.c:2633) [ 2175.888763] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2175.888967] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2175.889196] kthread (kernel/kthread.c:388) [ 2175.889370] ? __pfx_kthread (kernel/kthread.c:341) [ 2175.889572] ret_from_fork (arch/x86/kernel/process.c:147) [ 2175.889768] ? __pfx_kthread (kernel/kthread.c:341) [ 2175.889973] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2241.597875] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2198189 jiffies s: 6353 root: 0x8/. | [ 2241.598497] rcu: blocking rcu_node structures (internal RCU debug): | [ 2241.598832] Sending NMI from CPU 1 to CPUs 3: | [ 2241.599087] NMI backtrace for cpu 3 [ 2241.599093] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2241.599095] Workqueue: netns cleanup_net [ 2241.599102] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2241.599108] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2241.599110] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2241.599111] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2241.599112] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2241.599113] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2241.599114] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2241.599115] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2241.599118] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2241.599119] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2241.599120] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2241.599121] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2241.599122] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2241.599122] PKRU: 55555554 [ 2241.599124] Call Trace: [ 2241.599126] [ 2241.599129] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 2241.599134] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 2241.599139] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 2241.599142] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 2241.599146] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 2241.599148] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 2241.599153] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2241.599155] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2241.599157] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2241.599159] [ 2241.599160] [ 2241.599161] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2241.599166] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2241.599170] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2241.599174] ? up_read (kernel/locking/rwsem.c:1622) [ 2241.599177] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2241.599182] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2241.599184] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2241.599186] __tcf_action_put (net/sched/act_api.c:377) [ 2241.599192] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2241.599194] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2241.599196] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2241.599202] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2241.599207] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2241.599212] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2241.599215] tcf_chain_flush (net/sched/cls_api.c:737) [ 2241.599217] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2241.599220] tcf_block_put (net/sched/cls_api.c:1543) [ 2241.599223] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2241.599227] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2241.599230] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2241.599233] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2241.599237] default_device_exit_batch (net/core/dev.c:11629) [ 2241.599240] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2241.599242] ? process_one_work (kernel/workqueue.c:2606) [ 2241.599246] process_one_work (kernel/workqueue.c:2633) [ 2241.599249] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2241.599251] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2241.599253] kthread (kernel/kthread.c:388) [ 2241.599256] ? __pfx_kthread (kernel/kthread.c:341) [ 2241.599257] ret_from_fork (arch/x86/kernel/process.c:147) [ 2241.599262] ? __pfx_kthread (kernel/kthread.c:341) [ 2241.599263] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2241.599268] | [ 2253.888865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2253.889197] rcu: 3-....: (2209654 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=545926 | [ 2253.889723] rcu: (t=2210514 jiffies g=31081 q=2290 ncpus=4) [ 2253.890475] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2253.891067] Workqueue: netns cleanup_net [ 2253.891282] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2253.891576] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2253.892545] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2253.892823] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2253.893195] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2253.893569] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2253.893942] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2253.894315] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2253.894686] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2253.895110] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2253.895413] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2253.895786] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2253.896163] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2253.896536] PKRU: 55555554 [ 2253.896683] Call Trace: [ 2253.896819] [ 2253.896935] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2253.897174] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2253.897418] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 2253.897682] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 2253.897922] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2253.898163] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2253.898397] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2253.898679] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2253.898897] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2253.899152] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2253.899391] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2253.899621] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2253.899899] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2253.900167] [ 2253.900285] [ 2253.900407] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2253.900689] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2253.900950] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2253.901158] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2253.901358] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2253.901625] ? up_read (kernel/locking/rwsem.c:1622) [ 2253.901804] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2253.902056] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2253.902331] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2253.902588] __tcf_action_put (net/sched/act_api.c:377) [ 2253.902796] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2253.903010] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2253.903216] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2253.903499] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2253.903753] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2253.903999] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2253.904212] tcf_chain_flush (net/sched/cls_api.c:737) [ 2253.904421] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2253.904629] tcf_block_put (net/sched/cls_api.c:1543) [ 2253.904825] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2253.905065] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2253.905276] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2253.905471] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2253.905758] default_device_exit_batch (net/core/dev.c:11629) [ 2253.906015] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2253.906211] ? process_one_work (kernel/workqueue.c:2606) [ 2253.906437] process_one_work (kernel/workqueue.c:2633) [ 2253.906656] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2253.906859] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2253.907089] kthread (kernel/kthread.c:388) [ 2253.907262] ? __pfx_kthread (kernel/kthread.c:341) [ 2253.907465] ret_from_fork (arch/x86/kernel/process.c:147) [ 2253.907660] ? __pfx_kthread (kernel/kthread.c:341) [ 2253.907861] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2253.908078] | [ 2331.906866] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2331.907213] rcu: 3-....: (2287654 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=565225 | [ 2331.907741] rcu: (t=2288532 jiffies g=31081 q=2291 ncpus=4) [ 2331.908496] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2331.909091] Workqueue: netns cleanup_net [ 2331.909310] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2331.909608] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2331.910571] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2331.910849] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2331.911223] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2331.911606] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2331.911983] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2331.912358] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2331.912735] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2331.913158] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2331.913464] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2331.913840] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2331.914213] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2331.914588] PKRU: 55555554 [ 2331.914735] Call Trace: [ 2331.914874] [ 2331.914989] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2331.915231] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2331.915477] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 2331.915743] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 2331.915985] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2331.916228] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2331.916464] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2331.916747] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2331.916967] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2331.917223] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2331.917464] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2331.917696] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2331.917976] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2331.918244] [ 2331.918364] [ 2331.918483] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2331.918767] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2331.919029] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2331.919240] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2331.919440] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2331.919708] ? up_read (kernel/locking/rwsem.c:1622) [ 2331.919888] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2331.920138] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2331.920402] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2331.920660] __tcf_action_put (net/sched/act_api.c:377) [ 2331.920871] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2331.921087] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2331.921295] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2331.921586] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2331.921843] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2331.922088] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2331.922304] tcf_chain_flush (net/sched/cls_api.c:737) [ 2331.922512] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2331.922721] tcf_block_put (net/sched/cls_api.c:1543) [ 2331.922917] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2331.923160] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2331.923369] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2331.923564] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2331.923852] default_device_exit_batch (net/core/dev.c:11629) [ 2331.924110] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2331.924306] ? process_one_work (kernel/workqueue.c:2606) [ 2331.924538] process_one_work (kernel/workqueue.c:2633) [ 2331.924755] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2331.924958] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2331.925189] kthread (kernel/kthread.c:388) [ 2331.925368] ? __pfx_kthread (kernel/kthread.c:341) [ 2331.925569] ret_from_fork (arch/x86/kernel/process.c:147) [ 2331.925767] ? __pfx_kthread (kernel/kthread.c:341) [ 2331.925971] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2339.901901] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2296493 jiffies s: 6353 root: 0x8/. | [ 2339.902768] rcu: blocking rcu_node structures (internal RCU debug): | [ 2339.903261] Sending NMI from CPU 1 to CPUs 3: | [ 2339.903622] NMI backtrace for cpu 3 [ 2339.903627] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2339.903629] Workqueue: netns cleanup_net [ 2339.903632] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2339.903635] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2339.903637] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2339.903638] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2339.903639] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2339.903639] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2339.903640] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2339.903641] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2339.903643] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2339.903644] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2339.903645] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2339.903646] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2339.903646] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2339.903647] PKRU: 55555554 [ 2339.903647] Call Trace: [ 2339.903649] [ 2339.903650] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 2339.903655] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 2339.903659] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 2339.903662] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 2339.903664] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 2339.903666] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 2339.903669] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2339.903671] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2339.903673] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2339.903675] [ 2339.903676] [ 2339.903676] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2339.903680] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2339.903683] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2339.903685] ? up_read (kernel/locking/rwsem.c:1622) [ 2339.903687] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2339.903691] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2339.903692] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2339.903694] __tcf_action_put (net/sched/act_api.c:377) [ 2339.903698] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2339.903700] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2339.903701] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2339.903707] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2339.903712] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2339.903717] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2339.903720] tcf_chain_flush (net/sched/cls_api.c:737) [ 2339.903722] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2339.903725] tcf_block_put (net/sched/cls_api.c:1543) [ 2339.903728] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2339.903731] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2339.903734] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2339.903736] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2339.903738] default_device_exit_batch (net/core/dev.c:11629) [ 2339.903741] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2339.903743] ? process_one_work (kernel/workqueue.c:2606) [ 2339.903746] process_one_work (kernel/workqueue.c:2633) [ 2339.903748] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2339.903751] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2339.903752] kthread (kernel/kthread.c:388) [ 2339.903754] ? __pfx_kthread (kernel/kthread.c:341) [ 2339.903755] ret_from_fork (arch/x86/kernel/process.c:147) [ 2339.903758] ? __pfx_kthread (kernel/kthread.c:341) [ 2339.903759] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2339.903763] | [ 2409.924865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2409.925205] rcu: 3-....: (2365639 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=584342 | [ 2409.925735] rcu: (t=2366550 jiffies g=31081 q=2291 ncpus=4) [ 2409.926488] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2409.927080] Workqueue: netns cleanup_net [ 2409.927297] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2409.927593] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2409.928559] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2409.928835] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2409.929208] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2409.929581] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2409.929954] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2409.930326] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2409.930699] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2409.931128] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2409.931432] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2409.931804] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2409.932178] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2409.932551] PKRU: 55555554 [ 2409.932697] Call Trace: [ 2409.932833] [ 2409.932952] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2409.933193] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2409.933437] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 2409.933702] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 2409.933944] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2409.934185] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2409.934422] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2409.934707] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2409.934924] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2409.935179] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2409.935418] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2409.935647] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2409.935928] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2409.936196] [ 2409.936314] [ 2409.936432] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2409.936715] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2409.936976] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2409.937186] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2409.937387] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2409.937654] ? up_read (kernel/locking/rwsem.c:1622) [ 2409.937834] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2409.938076] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2409.938340] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2409.938596] __tcf_action_put (net/sched/act_api.c:377) [ 2409.938808] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2409.939023] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2409.939229] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2409.939510] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2409.939765] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2409.940011] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2409.940223] tcf_chain_flush (net/sched/cls_api.c:737) [ 2409.940431] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2409.940640] tcf_block_put (net/sched/cls_api.c:1543) [ 2409.940835] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2409.941080] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2409.941291] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2409.941486] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2409.941776] default_device_exit_batch (net/core/dev.c:11629) [ 2409.942033] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2409.942228] ? process_one_work (kernel/workqueue.c:2606) [ 2409.942453] process_one_work (kernel/workqueue.c:2633) [ 2409.942671] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2409.942874] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2409.943102] kthread (kernel/kthread.c:388) [ 2409.943278] ? __pfx_kthread (kernel/kthread.c:341) [ 2409.943479] ret_from_fork (arch/x86/kernel/process.c:147) [ 2409.943676] ? __pfx_kthread (kernel/kthread.c:341) [ 2409.943877] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2438.206876] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2394798 jiffies s: 6353 root: 0x8/. | [ 2438.207467] rcu: blocking rcu_node structures (internal RCU debug): | [ 2438.207792] Sending NMI from CPU 1 to CPUs 3: | [ 2438.208037] NMI backtrace for cpu 3 [ 2438.208042] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2438.208044] Workqueue: netns cleanup_net [ 2438.208049] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2438.208054] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2438.208055] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2438.208057] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2438.208058] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2438.208059] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2438.208060] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2438.208061] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2438.208063] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2438.208065] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2438.208065] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2438.208066] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2438.208067] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2438.208068] PKRU: 55555554 [ 2438.208068] Call Trace: [ 2438.208070] [ 2438.208072] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 2438.208078] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 2438.208082] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 2438.208085] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 2438.208088] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 2438.208090] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 2438.208094] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2438.208096] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2438.208098] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2438.208100] [ 2438.208101] [ 2438.208101] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2438.208106] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2438.208109] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2438.208113] ? up_read (kernel/locking/rwsem.c:1622) [ 2438.208116] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2438.208120] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2438.208121] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2438.208123] __tcf_action_put (net/sched/act_api.c:377) [ 2438.208128] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2438.208130] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2438.208132] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2438.208139] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2438.208144] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2438.208149] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2438.208152] tcf_chain_flush (net/sched/cls_api.c:737) [ 2438.208154] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2438.208157] tcf_block_put (net/sched/cls_api.c:1543) [ 2438.208161] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2438.208165] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2438.208169] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2438.208171] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2438.208175] default_device_exit_batch (net/core/dev.c:11629) [ 2438.208177] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2438.208180] ? process_one_work (kernel/workqueue.c:2606) [ 2438.208183] process_one_work (kernel/workqueue.c:2633) [ 2438.208186] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2438.208188] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2438.208190] kthread (kernel/kthread.c:388) [ 2438.208192] ? __pfx_kthread (kernel/kthread.c:341) [ 2438.208194] ret_from_fork (arch/x86/kernel/process.c:147) [ 2438.208197] ? __pfx_kthread (kernel/kthread.c:341) [ 2438.208198] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2438.208202] | [ 2487.942865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2487.943200] rcu: 3-....: (2443624 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=603475 | [ 2487.943726] rcu: (t=2444568 jiffies g=31081 q=2291 ncpus=4) [ 2487.944475] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2487.945065] Workqueue: netns cleanup_net [ 2487.945278] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2487.945574] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2487.946529] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2487.946807] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2487.947180] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2487.947552] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2487.947926] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2487.948296] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2487.948668] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2487.949091] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2487.949393] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2487.949769] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2487.950140] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2487.950512] PKRU: 55555554 [ 2487.950662] Call Trace: [ 2487.950798] [ 2487.950914] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2487.951152] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2487.951395] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 2487.951657] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 2487.951895] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2487.952136] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2487.952369] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2487.952649] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2487.952867] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2487.953121] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2487.953360] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2487.953593] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2487.953870] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2487.954134] [ 2487.954252] [ 2487.954369] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2487.954650] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2487.954910] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2487.955117] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2487.955314] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2487.955578] ? up_read (kernel/locking/rwsem.c:1622) [ 2487.955755] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2487.955996] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2487.956264] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2487.956519] __tcf_action_put (net/sched/act_api.c:377) [ 2487.956727] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2487.956942] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2487.957147] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2487.957428] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2487.957682] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2487.957931] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2487.958142] tcf_chain_flush (net/sched/cls_api.c:737) [ 2487.958350] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2487.958556] tcf_block_put (net/sched/cls_api.c:1543) [ 2487.958751] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2487.958994] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2487.959200] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2487.959394] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2487.959679] default_device_exit_batch (net/core/dev.c:11629) [ 2487.959935] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2487.960130] ? process_one_work (kernel/workqueue.c:2606) [ 2487.960355] process_one_work (kernel/workqueue.c:2633) [ 2487.960570] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2487.960778] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2487.961009] kthread (kernel/kthread.c:388) [ 2487.961180] ? __pfx_kthread (kernel/kthread.c:341) [ 2487.961381] ret_from_fork (arch/x86/kernel/process.c:147) [ 2487.961577] ? __pfx_kthread (kernel/kthread.c:341) [ 2487.961779] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2536.509882] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2493101 jiffies s: 6353 root: 0x8/. | [ 2536.510506] rcu: blocking rcu_node structures (internal RCU debug): | [ 2536.510836] Sending NMI from CPU 1 to CPUs 3: | [ 2536.511089] NMI backtrace for cpu 3 [ 2536.511097] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2536.511099] Workqueue: netns cleanup_net [ 2536.511107] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2536.511114] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2536.511116] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2536.511117] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2536.511118] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2536.511119] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2536.511120] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2536.511121] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2536.511125] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2536.511126] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2536.511127] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2536.511128] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2536.511129] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2536.511130] PKRU: 55555554 [ 2536.511130] Call Trace: [ 2536.511133] [ 2536.511136] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 2536.511141] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 2536.511146] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 2536.511148] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 2536.511153] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 2536.511155] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 2536.511160] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2536.511162] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2536.511164] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2536.511166] [ 2536.511166] [ 2536.511167] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2536.511173] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2536.511177] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2536.511183] ? up_read (kernel/locking/rwsem.c:1622) [ 2536.511186] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2536.511192] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2536.511194] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2536.511196] __tcf_action_put (net/sched/act_api.c:377) [ 2536.511203] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2536.511205] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2536.511208] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2536.511215] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2536.511220] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2536.511226] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2536.511229] tcf_chain_flush (net/sched/cls_api.c:737) [ 2536.511231] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2536.511234] tcf_block_put (net/sched/cls_api.c:1543) [ 2536.511238] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2536.511243] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2536.511247] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2536.511250] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2536.511255] default_device_exit_batch (net/core/dev.c:11629) [ 2536.511257] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2536.511260] ? process_one_work (kernel/workqueue.c:2606) [ 2536.511264] process_one_work (kernel/workqueue.c:2633) [ 2536.511267] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2536.511270] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2536.511271] kthread (kernel/kthread.c:388) [ 2536.511275] ? __pfx_kthread (kernel/kthread.c:341) [ 2536.511276] ret_from_fork (arch/x86/kernel/process.c:147) [ 2536.511279] ? __pfx_kthread (kernel/kthread.c:341) [ 2536.511281] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2536.511287] | [ 2565.960865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2565.961204] rcu: 3-....: (2521609 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=622374 | [ 2565.961731] rcu: (t=2522586 jiffies g=31081 q=2291 ncpus=4) [ 2565.962482] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2565.963076] Workqueue: netns cleanup_net [ 2565.963296] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2565.963590] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2565.964551] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2565.964846] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2565.965220] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2565.965593] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2565.965967] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2565.966342] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2565.966718] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2565.967140] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2565.967445] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2565.967821] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2565.968195] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2565.968568] PKRU: 55555554 [ 2565.968717] Call Trace: [ 2565.968855] [ 2565.968972] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2565.969211] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2565.969456] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 2565.969720] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 2565.969965] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2565.970204] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2565.970445] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2565.970725] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2565.970946] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2565.971201] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2565.971439] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2565.971671] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2565.971948] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2565.972215] [ 2565.972333] [ 2565.972452] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2565.972734] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2565.972996] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2565.973205] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2565.973406] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2565.973673] ? up_read (kernel/locking/rwsem.c:1622) [ 2565.973850] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2565.974095] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2565.974359] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2565.974616] __tcf_action_put (net/sched/act_api.c:377) [ 2565.974828] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2565.975044] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2565.975252] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2565.975533] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2565.975818] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2565.976080] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2565.976292] tcf_chain_flush (net/sched/cls_api.c:737) [ 2565.976530] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2565.976738] tcf_block_put (net/sched/cls_api.c:1543) [ 2565.976934] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2565.977186] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2565.977395] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2565.977598] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2565.977895] default_device_exit_batch (net/core/dev.c:11629) [ 2565.978167] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2565.978363] ? process_one_work (kernel/workqueue.c:2606) [ 2565.978607] process_one_work (kernel/workqueue.c:2633) [ 2565.978836] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2565.979040] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2565.979277] kthread (kernel/kthread.c:388) [ 2565.979465] ? __pfx_kthread (kernel/kthread.c:341) [ 2565.979668] ret_from_fork (arch/x86/kernel/process.c:147) [ 2565.979894] ? __pfx_kthread (kernel/kthread.c:341) [ 2565.980096] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2634.813883] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2591405 jiffies s: 6353 root: 0x8/. | [ 2634.814511] rcu: blocking rcu_node structures (internal RCU debug): | [ 2634.814840] Sending NMI from CPU 1 to CPUs 3: | [ 2634.815092] NMI backtrace for cpu 3 [ 2634.815099] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2634.815101] Workqueue: netns cleanup_net [ 2634.815109] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2634.815116] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2634.815118] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2634.815119] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2634.815121] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2634.815122] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2634.815123] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2634.815124] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2634.815126] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2634.815127] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2634.815128] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2634.815129] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2634.815130] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2634.815130] PKRU: 55555554 [ 2634.815131] Call Trace: [ 2634.815133] [ 2634.815136] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 2634.815141] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 2634.815146] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 2634.815149] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 2634.815154] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 2634.815156] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 2634.815161] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2634.815163] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2634.815165] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2634.815167] [ 2634.815168] [ 2634.815169] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2634.815175] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2634.815179] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2634.815185] ? up_read (kernel/locking/rwsem.c:1622) [ 2634.815189] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2634.815194] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2634.815196] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2634.815198] __tcf_action_put (net/sched/act_api.c:377) [ 2634.815205] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2634.815207] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2634.815210] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2634.815217] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2634.815223] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2634.815228] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2634.815231] tcf_chain_flush (net/sched/cls_api.c:737) [ 2634.815234] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2634.815237] tcf_block_put (net/sched/cls_api.c:1543) [ 2634.815241] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2634.815245] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2634.815250] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2634.815252] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2634.815257] default_device_exit_batch (net/core/dev.c:11629) [ 2634.815260] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2634.815262] ? process_one_work (kernel/workqueue.c:2606) [ 2634.815266] process_one_work (kernel/workqueue.c:2633) [ 2634.815268] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2634.815271] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2634.815273] kthread (kernel/kthread.c:388) [ 2634.815275] ? __pfx_kthread (kernel/kthread.c:341) [ 2634.815277] ret_from_fork (arch/x86/kernel/process.c:147) [ 2634.815281] ? __pfx_kthread (kernel/kthread.c:341) [ 2634.815282] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2634.815288] | [ 2643.978866] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2643.979209] rcu: 3-....: (2599594 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=640579 | [ 2643.979741] rcu: (t=2600604 jiffies g=31081 q=2291 ncpus=4) [ 2643.980513] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2643.981111] Workqueue: netns cleanup_net [ 2643.981333] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2643.981634] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2643.982595] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2643.982879] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2643.983253] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2643.983628] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2643.984009] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2643.984387] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2643.984764] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2643.985190] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2643.985494] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2643.985872] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2643.986248] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2643.986625] PKRU: 55555554 [ 2643.986772] Call Trace: [ 2643.986910] [ 2643.987027] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2643.987268] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2643.987514] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 2643.987779] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 2643.988023] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2643.988266] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2643.988500] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2643.988784] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2643.989003] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2643.989259] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2643.989499] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2643.989728] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2643.990017] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2643.990283] [ 2643.990401] [ 2643.990518] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2643.990803] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2643.991063] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2643.991275] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2643.991476] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2643.991743] ? up_read (kernel/locking/rwsem.c:1622) [ 2643.991924] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2643.992171] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2643.992443] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2643.992698] __tcf_action_put (net/sched/act_api.c:377) [ 2643.992912] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2643.993127] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2643.993334] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2643.993622] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2643.993884] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2643.994132] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2643.994343] tcf_chain_flush (net/sched/cls_api.c:737) [ 2643.994551] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2643.994760] tcf_block_put (net/sched/cls_api.c:1543) [ 2643.994957] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2643.995196] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2643.995413] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2643.995609] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2643.995900] default_device_exit_batch (net/core/dev.c:11629) [ 2643.996159] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2643.996354] ? process_one_work (kernel/workqueue.c:2606) [ 2643.996583] process_one_work (kernel/workqueue.c:2633) [ 2643.996802] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2643.997012] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2643.997242] kthread (kernel/kthread.c:388) [ 2643.997417] ? __pfx_kthread (kernel/kthread.c:341) [ 2643.997620] ret_from_fork (arch/x86/kernel/process.c:147) [ 2643.997816] ? __pfx_kthread (kernel/kthread.c:341) [ 2643.998029] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2643.998250] | [ 2721.996866] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2721.997211] rcu: 3-....: (2677594 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=658980 | [ 2721.997743] rcu: (t=2678622 jiffies g=31081 q=2291 ncpus=4) [ 2721.998504] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2721.999103] Workqueue: netns cleanup_net [ 2721.999321] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2721.999625] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2722.000587] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2722.000866] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2722.001238] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2722.001613] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2722.001988] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2722.002386] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2722.002761] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2722.003188] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2722.003493] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2722.003867] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2722.004242] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2722.004617] PKRU: 55555554 [ 2722.004764] Call Trace: [ 2722.004901] [ 2722.005018] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2722.005259] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2722.005504] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 2722.005770] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 2722.006011] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2722.006254] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2722.006490] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2722.006773] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2722.006991] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2722.007247] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2722.007487] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2722.007719] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2722.007999] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2722.008266] [ 2722.008384] [ 2722.008502] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2722.008786] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2722.009047] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2722.009257] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2722.009463] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2722.009731] ? up_read (kernel/locking/rwsem.c:1622) [ 2722.009912] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2722.010155] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2722.010419] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2722.010675] __tcf_action_put (net/sched/act_api.c:377) [ 2722.010888] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2722.011103] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2722.011309] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2722.011593] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2722.011849] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2722.012096] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2722.012312] tcf_chain_flush (net/sched/cls_api.c:737) [ 2722.012521] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2722.012729] tcf_block_put (net/sched/cls_api.c:1543) [ 2722.012926] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2722.013166] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2722.013381] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2722.013577] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2722.013868] default_device_exit_batch (net/core/dev.c:11629) [ 2722.014126] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2722.014323] ? process_one_work (kernel/workqueue.c:2606) [ 2722.014553] process_one_work (kernel/workqueue.c:2633) [ 2722.014772] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2722.014976] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2722.015206] kthread (kernel/kthread.c:388) [ 2722.015381] ? __pfx_kthread (kernel/kthread.c:341) [ 2722.015584] ret_from_fork (arch/x86/kernel/process.c:147) [ 2722.015780] ? __pfx_kthread (kernel/kthread.c:341) [ 2722.015982] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2733.117882] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2689709 jiffies s: 6353 root: 0x8/. | [ 2733.118526] rcu: blocking rcu_node structures (internal RCU debug): | [ 2733.118857] Sending NMI from CPU 1 to CPUs 3: | [ 2733.119110] NMI backtrace for cpu 3 [ 2733.119118] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2733.119120] Workqueue: netns cleanup_net [ 2733.119129] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2733.119135] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2733.119137] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2733.119138] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2733.119139] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2733.119140] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2733.119141] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2733.119143] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2733.119145] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2733.119147] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2733.119148] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2733.119149] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2733.119150] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2733.119150] PKRU: 55555554 [ 2733.119151] Call Trace: [ 2733.119154] [ 2733.119157] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 2733.119162] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 2733.119166] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 2733.119169] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 2733.119173] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 2733.119175] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 2733.119180] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2733.119182] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2733.119184] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2733.119186] [ 2733.119187] [ 2733.119188] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2733.119193] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2733.119197] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2733.119203] ? up_read (kernel/locking/rwsem.c:1622) [ 2733.119207] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2733.119212] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2733.119213] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2733.119216] __tcf_action_put (net/sched/act_api.c:377) [ 2733.119222] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2733.119224] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2733.119227] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2733.119234] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2733.119241] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2733.119246] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2733.119249] tcf_chain_flush (net/sched/cls_api.c:737) [ 2733.119251] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2733.119254] tcf_block_put (net/sched/cls_api.c:1543) [ 2733.119258] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2733.119262] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2733.119267] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2733.119270] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2733.119274] default_device_exit_batch (net/core/dev.c:11629) [ 2733.119277] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2733.119280] ? process_one_work (kernel/workqueue.c:2606) [ 2733.119283] process_one_work (kernel/workqueue.c:2633) [ 2733.119286] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2733.119289] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2733.119291] kthread (kernel/kthread.c:388) [ 2733.119293] ? __pfx_kthread (kernel/kthread.c:341) [ 2733.119295] ret_from_fork (arch/x86/kernel/process.c:147) [ 2733.119299] ? __pfx_kthread (kernel/kthread.c:341) [ 2733.119300] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2733.119306] | [ 2800.014866] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2800.015206] rcu: 3-....: (2755579 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=677861 | [ 2800.015734] rcu: (t=2756640 jiffies g=31081 q=2291 ncpus=4) [ 2800.016491] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2800.017090] Workqueue: netns cleanup_net [ 2800.017309] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2800.017603] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2800.018572] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2800.018849] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2800.019231] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2800.019608] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2800.019984] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2800.020361] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2800.020740] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2800.021167] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2800.021473] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2800.021850] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2800.022227] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2800.022601] PKRU: 55555554 [ 2800.022752] Call Trace: [ 2800.022890] [ 2800.023007] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2800.023245] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2800.023494] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 2800.023759] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 2800.024004] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2800.024246] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2800.024482] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2800.024765] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2800.024988] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2800.025245] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2800.025486] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2800.025719] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2800.025997] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2800.026267] [ 2800.026387] [ 2800.026507] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2800.026791] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2800.027055] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2800.027269] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2800.027472] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2800.027741] ? up_read (kernel/locking/rwsem.c:1622) [ 2800.027922] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2800.028164] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2800.028430] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2800.028687] __tcf_action_put (net/sched/act_api.c:377) [ 2800.028906] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2800.029131] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2800.029338] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2800.029620] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2800.029877] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2800.030126] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2800.030337] tcf_chain_flush (net/sched/cls_api.c:737) [ 2800.030545] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2800.030753] tcf_block_put (net/sched/cls_api.c:1543) [ 2800.030951] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2800.031198] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2800.031406] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2800.031602] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2800.031892] default_device_exit_batch (net/core/dev.c:11629) [ 2800.032149] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2800.032346] ? process_one_work (kernel/workqueue.c:2606) [ 2800.032571] process_one_work (kernel/workqueue.c:2633) [ 2800.032787] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2800.032992] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2800.033222] kthread (kernel/kthread.c:388) [ 2800.033397] ? __pfx_kthread (kernel/kthread.c:341) [ 2800.033599] ret_from_fork (arch/x86/kernel/process.c:147) [ 2800.033796] ? __pfx_kthread (kernel/kthread.c:341) [ 2800.033999] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2831.421878] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2788013 jiffies s: 6353 root: 0x8/. | [ 2831.422488] rcu: blocking rcu_node structures (internal RCU debug): | [ 2831.422819] Sending NMI from CPU 1 to CPUs 3: | [ 2831.423082] NMI backtrace for cpu 3 [ 2831.423089] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2831.423091] Workqueue: netns cleanup_net [ 2831.423097] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 2831.423103] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 <85> c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: f3 0f 1e fa endbr64 12: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 17: 41 55 push %r13 19: 41 54 push %r12 1b: 55 push %rbp 1c: 48 89 fd mov %rdi,%rbp 1f: 53 push %rbx 20: 66 90 xchg %ax,%ax 22: ba 01 00 00 00 mov $0x1,%edx 27: 8b 45 00 mov 0x0(%rbp),%eax 2a:* 85 c0 test %eax,%eax <-- trapping instruction 2c: 75 14 jne 0x42 2e: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 33: 85 c0 test %eax,%eax 35: 75 f0 jne 0x27 37: 5b pop %rbx 38: 5d pop %rbp 39: 41 5c pop %r12 3b: 41 5d pop %r13 3d: c3 ret 3e: cc int3 3f: cc int3 Code starting with the faulting instruction =========================================== 0: 85 c0 test %eax,%eax 2: 75 14 jne 0x18 4: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 9: 85 c0 test %eax,%eax b: 75 f0 jne 0xfffffffffffffffd d: 5b pop %rbx e: 5d pop %rbp f: 41 5c pop %r12 11: 41 5d pop %r13 13: c3 ret 14: cc int3 15: cc int3 [ 2831.423105] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2831.423106] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2831.423108] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2831.423108] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2831.423109] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2831.423110] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2831.423113] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2831.423114] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2831.423115] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2831.423116] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2831.423117] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2831.423118] PKRU: 55555554 [ 2831.423118] Call Trace: [ 2831.423121] [ 2831.423124] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 2831.423129] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 2831.423133] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 2831.423136] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 2831.423140] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 2831.423141] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 2831.423145] ? queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 2831.423147] ? queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 2831.423150] ? queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 2831.423151] [ 2831.423152] [ 2831.423153] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2831.423158] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2831.423161] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2831.423164] ? up_read (kernel/locking/rwsem.c:1622) [ 2831.423167] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2831.423172] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2831.423173] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2831.423176] __tcf_action_put (net/sched/act_api.c:377) [ 2831.423181] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2831.423183] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2831.423185] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2831.423191] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2831.423197] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2831.423202] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2831.423204] tcf_chain_flush (net/sched/cls_api.c:737) [ 2831.423207] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2831.423210] tcf_block_put (net/sched/cls_api.c:1543) [ 2831.423222] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2831.423226] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2831.423229] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2831.423231] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2831.423235] default_device_exit_batch (net/core/dev.c:11629) [ 2831.423238] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2831.423240] ? process_one_work (kernel/workqueue.c:2606) [ 2831.423244] process_one_work (kernel/workqueue.c:2633) [ 2831.423247] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2831.423249] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2831.423251] kthread (kernel/kthread.c:388) [ 2831.423253] ? __pfx_kthread (kernel/kthread.c:341) [ 2831.423255] ret_from_fork (arch/x86/kernel/process.c:147) [ 2831.423259] ? __pfx_kthread (kernel/kthread.c:341) [ 2831.423260] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2831.423265] | [ 2878.032866] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2878.033199] rcu: 3-....: (2833564 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=696977 | [ 2878.033727] rcu: (t=2834658 jiffies g=31081 q=2291 ncpus=4) [ 2878.034482] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2878.035078] Workqueue: netns cleanup_net [ 2878.035295] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2878.035589] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2878.036553] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2878.036833] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2878.037210] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2878.037585] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2878.037961] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2878.038335] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2878.038718] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2878.039140] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2878.039446] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2878.039821] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2878.040197] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2878.040570] PKRU: 55555554 [ 2878.040720] Call Trace: [ 2878.040857] [ 2878.040975] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2878.041212] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2878.041458] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 2878.041724] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 2878.041968] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2878.042210] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2878.042443] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2878.042727] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2878.042946] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2878.043203] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2878.043441] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2878.043673] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2878.043951] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2878.044218] [ 2878.044337] [ 2878.044455] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2878.044739] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2878.045001] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2878.045211] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2878.045412] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2878.045679] ? up_read (kernel/locking/rwsem.c:1622) [ 2878.045857] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2878.046100] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2878.046363] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2878.046620] __tcf_action_put (net/sched/act_api.c:377) [ 2878.046829] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2878.047048] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2878.047256] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2878.047538] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2878.047794] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2878.048043] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2878.048255] tcf_chain_flush (net/sched/cls_api.c:737) [ 2878.048461] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2878.048675] tcf_block_put (net/sched/cls_api.c:1543) [ 2878.048871] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2878.049114] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2878.049321] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2878.049517] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2878.049806] default_device_exit_batch (net/core/dev.c:11629) [ 2878.050064] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2878.050259] ? process_one_work (kernel/workqueue.c:2606) [ 2878.050485] process_one_work (kernel/workqueue.c:2633) [ 2878.050701] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2878.050905] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2878.051136] kthread (kernel/kthread.c:388) [ 2878.051311] ? __pfx_kthread (kernel/kthread.c:341) [ 2878.051514] ret_from_fork (arch/x86/kernel/process.c:147) [ 2878.051710] ? __pfx_kthread (kernel/kthread.c:341) [ 2878.051912] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2929.726877] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2886318 jiffies s: 6353 root: 0x8/. | [ 2929.727486] rcu: blocking rcu_node structures (internal RCU debug): | [ 2929.727823] Sending NMI from CPU 1 to CPUs 3: | [ 2929.728081] NMI backtrace for cpu 3 [ 2929.728089] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2929.728091] Workqueue: netns cleanup_net [ 2929.728099] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2929.728104] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2929.728106] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2929.728108] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2929.728109] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2929.728110] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2929.728110] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2929.728111] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2929.728114] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2929.728115] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2929.728116] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2929.728117] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2929.728118] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2929.728119] PKRU: 55555554 [ 2929.728119] Call Trace: [ 2929.728123] [ 2929.728125] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 2929.728129] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 2929.728134] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 2929.728137] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 2929.728141] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 2929.728143] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 2929.728147] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2929.728149] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2929.728151] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2929.728153] [ 2929.728154] [ 2929.728155] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2929.728160] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2929.728165] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2929.728169] ? up_read (kernel/locking/rwsem.c:1622) [ 2929.728172] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2929.728177] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2929.728178] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2929.728180] __tcf_action_put (net/sched/act_api.c:377) [ 2929.728186] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2929.728188] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2929.728190] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2929.728197] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2929.728203] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2929.728208] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2929.728211] tcf_chain_flush (net/sched/cls_api.c:737) [ 2929.728213] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2929.728216] tcf_block_put (net/sched/cls_api.c:1543) [ 2929.728219] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2929.728223] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2929.728226] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2929.728229] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2929.728232] default_device_exit_batch (net/core/dev.c:11629) [ 2929.728235] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2929.728237] ? process_one_work (kernel/workqueue.c:2606) [ 2929.728241] process_one_work (kernel/workqueue.c:2633) [ 2929.728244] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2929.728246] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2929.728248] kthread (kernel/kthread.c:388) [ 2929.728251] ? __pfx_kthread (kernel/kthread.c:341) [ 2929.728252] ret_from_fork (arch/x86/kernel/process.c:147) [ 2929.728256] ? __pfx_kthread (kernel/kthread.c:341) [ 2929.728257] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 2929.728262] | [ 2956.050866] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 2956.051198] rcu: 3-....: (2911549 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=715818 | [ 2956.051726] rcu: (t=2912676 jiffies g=31081 q=2291 ncpus=4) [ 2956.052483] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 2956.053083] Workqueue: netns cleanup_net [ 2956.053297] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2956.053591] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 2956.054549] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 2956.054826] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 2956.055201] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 2956.055575] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 2956.055949] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 2956.056322] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 2956.056695] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 2956.057121] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 2956.057424] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 2956.057798] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 2956.058177] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 2956.058552] PKRU: 55555554 [ 2956.058699] Call Trace: [ 2956.058836] [ 2956.058953] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 2956.059193] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 2956.059436] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 2956.059700] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 2956.059941] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 2956.060182] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 2956.060416] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 2956.060700] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 2956.060919] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 2956.061178] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 2956.061418] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 2956.061646] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 2956.061926] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 2956.062192] [ 2956.062310] [ 2956.062427] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 2956.062712] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 2956.062974] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 2956.063182] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 2956.063382] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2956.063649] ? up_read (kernel/locking/rwsem.c:1622) [ 2956.063828] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 2956.064069] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2956.064333] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 2956.064589] __tcf_action_put (net/sched/act_api.c:377) [ 2956.064807] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 2956.065023] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 2956.065231] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 2956.065513] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 2956.065767] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 2956.066015] tcf_proto_destroy (net/sched/cls_api.c:418) [ 2956.066226] tcf_chain_flush (net/sched/cls_api.c:737) [ 2956.066435] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 2956.066643] tcf_block_put (net/sched/cls_api.c:1543) [ 2956.066839] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 2956.067079] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 2956.067290] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 2956.067485] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 2956.067771] default_device_exit_batch (net/core/dev.c:11629) [ 2956.068028] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 2956.068231] ? process_one_work (kernel/workqueue.c:2606) [ 2956.068456] process_one_work (kernel/workqueue.c:2633) [ 2956.068675] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 2956.068879] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 2956.069107] kthread (kernel/kthread.c:388) [ 2956.069282] ? __pfx_kthread (kernel/kthread.c:341) [ 2956.069484] ret_from_fork (arch/x86/kernel/process.c:147) [ 2956.069680] ? __pfx_kthread (kernel/kthread.c:341) [ 2956.069882] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3028.029890] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 2984621 jiffies s: 6353 root: 0x8/. | [ 3028.030501] rcu: blocking rcu_node structures (internal RCU debug): | [ 3028.030835] Sending NMI from CPU 1 to CPUs 3: | [ 3028.031091] NMI backtrace for cpu 3 [ 3028.031101] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3028.031104] Workqueue: netns cleanup_net [ 3028.031113] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3028.031119] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3028.031121] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3028.031122] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3028.031124] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3028.031124] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3028.031125] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3028.031126] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3028.031129] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3028.031130] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3028.031131] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3028.031132] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3028.031133] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3028.031134] PKRU: 55555554 [ 3028.031134] Call Trace: [ 3028.031142] [ 3028.031147] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 3028.031152] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 3028.031157] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 3028.031160] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 3028.031164] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 3028.031166] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 3028.031170] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3028.031173] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3028.031175] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3028.031177] [ 3028.031177] [ 3028.031178] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3028.031183] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3028.031188] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3028.031192] ? up_read (kernel/locking/rwsem.c:1622) [ 3028.031195] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3028.031200] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3028.031202] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3028.031204] __tcf_action_put (net/sched/act_api.c:377) [ 3028.031210] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3028.031212] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3028.031214] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3028.031220] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3028.031226] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3028.031232] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3028.031235] tcf_chain_flush (net/sched/cls_api.c:737) [ 3028.031237] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3028.031241] tcf_block_put (net/sched/cls_api.c:1543) [ 3028.031244] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3028.031248] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3028.031253] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3028.031255] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3028.031259] default_device_exit_batch (net/core/dev.c:11629) [ 3028.031262] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3028.031264] ? process_one_work (kernel/workqueue.c:2606) [ 3028.031269] process_one_work (kernel/workqueue.c:2633) [ 3028.031272] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3028.031274] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3028.031276] kthread (kernel/kthread.c:388) [ 3028.031279] ? __pfx_kthread (kernel/kthread.c:341) [ 3028.031281] ret_from_fork (arch/x86/kernel/process.c:147) [ 3028.031285] ? __pfx_kthread (kernel/kthread.c:341) [ 3028.031287] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3028.031292] | [ 3034.068866] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3034.069196] rcu: 3-....: (2989534 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=734494 | [ 3034.069723] rcu: (t=2990694 jiffies g=31081 q=2291 ncpus=4) [ 3034.070476] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3034.071075] Workqueue: netns cleanup_net [ 3034.071293] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3034.071585] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3034.072547] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3034.072823] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3034.073198] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3034.073571] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3034.073945] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3034.074319] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3034.074695] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3034.075118] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3034.075422] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3034.075797] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3034.076170] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3034.076542] PKRU: 55555554 [ 3034.076692] Call Trace: [ 3034.076830] [ 3034.076945] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3034.077182] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3034.077426] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 3034.077690] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 3034.077941] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3034.078179] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3034.078415] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3034.078700] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3034.078922] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3034.079177] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3034.079415] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3034.079646] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3034.079925] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3034.080191] [ 3034.080313] [ 3034.080432] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3034.080715] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3034.080976] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3034.081188] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3034.081390] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3034.081657] ? up_read (kernel/locking/rwsem.c:1622) [ 3034.081834] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3034.082076] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3034.082340] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3034.082596] __tcf_action_put (net/sched/act_api.c:377) [ 3034.082805] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3034.083020] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3034.083228] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3034.083510] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3034.083766] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3034.084012] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3034.084225] tcf_chain_flush (net/sched/cls_api.c:737) [ 3034.084433] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3034.084639] tcf_block_put (net/sched/cls_api.c:1543) [ 3034.084834] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3034.085077] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3034.085284] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3034.085478] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3034.085764] default_device_exit_batch (net/core/dev.c:11629) [ 3034.086021] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3034.086215] ? process_one_work (kernel/workqueue.c:2606) [ 3034.086443] process_one_work (kernel/workqueue.c:2633) [ 3034.086659] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3034.086861] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3034.087093] kthread (kernel/kthread.c:388) [ 3034.087266] ? __pfx_kthread (kernel/kthread.c:341) [ 3034.087469] ret_from_fork (arch/x86/kernel/process.c:147) [ 3034.087669] ? __pfx_kthread (kernel/kthread.c:341) [ 3034.087879] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3034.088093] | [ 3112.086868] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3112.087225] rcu: 3-....: (3067534 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=752849 | [ 3112.087757] rcu: (t=3068712 jiffies g=31081 q=2291 ncpus=4) [ 3112.088514] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3112.089117] Workqueue: netns cleanup_net [ 3112.089338] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3112.089635] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3112.090601] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3112.090880] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3112.091254] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3112.091632] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3112.092008] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3112.092381] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3112.092755] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3112.093185] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3112.093489] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3112.093865] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3112.094245] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3112.094619] PKRU: 55555554 [ 3112.094765] Call Trace: [ 3112.094904] [ 3112.095020] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3112.095261] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3112.095505] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 3112.095769] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 3112.096010] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3112.096251] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3112.096488] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3112.096773] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3112.096993] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3112.097250] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3112.097503] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3112.097740] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3112.098025] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3112.098291] [ 3112.098410] [ 3112.098528] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3112.098810] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3112.099072] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3112.099280] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3112.099486] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3112.099752] ? up_read (kernel/locking/rwsem.c:1622) [ 3112.099931] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3112.100172] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3112.100435] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3112.100691] __tcf_action_put (net/sched/act_api.c:377) [ 3112.100908] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3112.101122] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3112.101326] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3112.101609] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3112.101863] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3112.102114] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3112.102328] tcf_chain_flush (net/sched/cls_api.c:737) [ 3112.102536] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3112.102744] tcf_block_put (net/sched/cls_api.c:1543) [ 3112.102942] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3112.103183] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3112.103392] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3112.103587] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3112.103874] default_device_exit_batch (net/core/dev.c:11629) [ 3112.104130] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3112.104325] ? process_one_work (kernel/workqueue.c:2606) [ 3112.104552] process_one_work (kernel/workqueue.c:2633) [ 3112.104769] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3112.104972] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3112.105202] kthread (kernel/kthread.c:388) [ 3112.105375] ? __pfx_kthread (kernel/kthread.c:341) [ 3112.105576] ret_from_fork (arch/x86/kernel/process.c:147) [ 3112.105772] ? __pfx_kthread (kernel/kthread.c:341) [ 3112.105975] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3126.333878] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 3082925 jiffies s: 6353 root: 0x8/. | [ 3126.334484] rcu: blocking rcu_node structures (internal RCU debug): | [ 3126.334852] Sending NMI from CPU 1 to CPUs 3: | [ 3126.335107] NMI backtrace for cpu 3 [ 3126.335115] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3126.335117] Workqueue: netns cleanup_net [ 3126.335126] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3126.335133] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3126.335135] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3126.335136] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3126.335138] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3126.335139] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3126.335140] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3126.335141] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3126.335144] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3126.335145] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3126.335146] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3126.335147] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3126.335148] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3126.335149] PKRU: 55555554 [ 3126.335149] Call Trace: [ 3126.335152] [ 3126.335155] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 3126.335159] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 3126.335165] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 3126.335168] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 3126.335172] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 3126.335174] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 3126.335178] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3126.335181] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3126.335183] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3126.335185] [ 3126.335185] [ 3126.335186] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3126.335192] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3126.335196] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3126.335201] ? up_read (kernel/locking/rwsem.c:1622) [ 3126.335204] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3126.335209] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3126.335211] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3126.335213] __tcf_action_put (net/sched/act_api.c:377) [ 3126.335219] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3126.335222] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3126.335224] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3126.335231] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3126.335237] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3126.335242] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3126.335245] tcf_chain_flush (net/sched/cls_api.c:737) [ 3126.335247] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3126.335251] tcf_block_put (net/sched/cls_api.c:1543) [ 3126.335254] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3126.335259] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3126.335263] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3126.335266] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3126.335270] default_device_exit_batch (net/core/dev.c:11629) [ 3126.335273] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3126.335276] ? process_one_work (kernel/workqueue.c:2606) [ 3126.335280] process_one_work (kernel/workqueue.c:2633) [ 3126.335283] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3126.335286] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3126.335288] kthread (kernel/kthread.c:388) [ 3126.335291] ? __pfx_kthread (kernel/kthread.c:341) [ 3126.335292] ret_from_fork (arch/x86/kernel/process.c:147) [ 3126.335297] ? __pfx_kthread (kernel/kthread.c:341) [ 3126.335298] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3126.335303] | [ 3190.104866] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3190.105211] rcu: 3-....: (3145519 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=771033 | [ 3190.105736] rcu: (t=3146730 jiffies g=31081 q=2291 ncpus=4) [ 3190.106487] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3190.107085] Workqueue: netns cleanup_net [ 3190.107303] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3190.107595] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3190.108553] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3190.108828] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3190.109203] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3190.109575] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3190.109949] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3190.110323] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3190.110699] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3190.111125] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3190.111441] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3190.111813] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3190.112188] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3190.112561] PKRU: 55555554 [ 3190.112711] Call Trace: [ 3190.112848] [ 3190.112965] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3190.113201] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3190.113446] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 3190.113710] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 3190.113954] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3190.114192] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3190.114426] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3190.114706] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3190.114928] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3190.115183] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3190.115420] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3190.115651] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3190.115928] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3190.116194] [ 3190.116311] [ 3190.116430] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3190.116710] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3190.116979] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3190.117190] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3190.117390] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3190.117657] ? up_read (kernel/locking/rwsem.c:1622) [ 3190.117833] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3190.118076] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3190.118339] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3190.118594] __tcf_action_put (net/sched/act_api.c:377) [ 3190.118808] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3190.119025] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3190.119232] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3190.119514] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3190.119769] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3190.120016] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3190.120228] tcf_chain_flush (net/sched/cls_api.c:737) [ 3190.120434] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3190.120640] tcf_block_put (net/sched/cls_api.c:1543) [ 3190.120835] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3190.121077] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3190.121285] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3190.121479] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3190.121766] default_device_exit_batch (net/core/dev.c:11629) [ 3190.122023] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3190.122217] ? process_one_work (kernel/workqueue.c:2606) [ 3190.122443] process_one_work (kernel/workqueue.c:2633) [ 3190.122658] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3190.122860] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3190.123091] kthread (kernel/kthread.c:388) [ 3190.123265] ? __pfx_kthread (kernel/kthread.c:341) [ 3190.123466] ret_from_fork (arch/x86/kernel/process.c:147) [ 3190.123663] ? __pfx_kthread (kernel/kthread.c:341) [ 3190.123864] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3224.637877] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 3181229 jiffies s: 6353 root: 0x8/. | [ 3224.638693] rcu: blocking rcu_node structures (internal RCU debug): | [ 3224.639153] Sending NMI from CPU 1 to CPUs 3: | [ 3224.639495] NMI backtrace for cpu 3 [ 3224.639509] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3224.639513] Workqueue: netns cleanup_net [ 3224.639523] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 3224.639529] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 <85> c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: f3 0f 1e fa endbr64 12: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 17: 41 55 push %r13 19: 41 54 push %r12 1b: 55 push %rbp 1c: 48 89 fd mov %rdi,%rbp 1f: 53 push %rbx 20: 66 90 xchg %ax,%ax 22: ba 01 00 00 00 mov $0x1,%edx 27: 8b 45 00 mov 0x0(%rbp),%eax 2a:* 85 c0 test %eax,%eax <-- trapping instruction 2c: 75 14 jne 0x42 2e: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 33: 85 c0 test %eax,%eax 35: 75 f0 jne 0x27 37: 5b pop %rbx 38: 5d pop %rbp 39: 41 5c pop %r12 3b: 41 5d pop %r13 3d: c3 ret 3e: cc int3 3f: cc int3 Code starting with the faulting instruction =========================================== 0: 85 c0 test %eax,%eax 2: 75 14 jne 0x18 4: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 9: 85 c0 test %eax,%eax b: 75 f0 jne 0xfffffffffffffffd d: 5b pop %rbx e: 5d pop %rbp f: 41 5c pop %r12 11: 41 5d pop %r13 13: c3 ret 14: cc int3 15: cc int3 [ 3224.639531] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3224.639532] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3224.639534] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3224.639535] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3224.639536] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3224.639537] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3224.639539] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3224.639540] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3224.639541] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3224.639542] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3224.639543] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3224.639543] PKRU: 55555554 [ 3224.639544] Call Trace: [ 3224.639549] [ 3224.639557] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 3224.639563] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 3224.639567] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 3224.639570] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 3224.639574] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 3224.639575] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 3224.639580] ? queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 3224.639582] ? queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 3224.639585] ? queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 3224.639587] [ 3224.639587] [ 3224.639588] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3224.639593] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3224.639597] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3224.639601] ? up_read (kernel/locking/rwsem.c:1622) [ 3224.639604] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3224.639608] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3224.639610] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3224.639612] __tcf_action_put (net/sched/act_api.c:377) [ 3224.639620] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3224.639622] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3224.639624] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3224.639630] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3224.639636] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3224.639641] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3224.639643] tcf_chain_flush (net/sched/cls_api.c:737) [ 3224.639645] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3224.639648] tcf_block_put (net/sched/cls_api.c:1543) [ 3224.639651] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3224.639654] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3224.639657] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3224.639660] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3224.639663] default_device_exit_batch (net/core/dev.c:11629) [ 3224.639665] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3224.639668] ? process_one_work (kernel/workqueue.c:2606) [ 3224.639671] process_one_work (kernel/workqueue.c:2633) [ 3224.639674] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3224.639677] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3224.639679] kthread (kernel/kthread.c:388) [ 3224.639681] ? __pfx_kthread (kernel/kthread.c:341) [ 3224.639683] ret_from_fork (arch/x86/kernel/process.c:147) [ 3224.639687] ? __pfx_kthread (kernel/kthread.c:341) [ 3224.639688] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3224.639692] | [ 3268.122866] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3268.123204] rcu: 3-....: (3223504 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=789830 | [ 3268.123730] rcu: (t=3224748 jiffies g=31081 q=2291 ncpus=4) [ 3268.124483] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3268.125080] Workqueue: netns cleanup_net [ 3268.125298] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3268.125592] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3268.126557] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3268.126833] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3268.127207] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3268.127579] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3268.127952] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3268.128325] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3268.128699] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3268.129121] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3268.129425] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3268.129801] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3268.130174] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3268.130546] PKRU: 55555554 [ 3268.130696] Call Trace: [ 3268.130833] [ 3268.130950] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3268.131188] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3268.131434] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 3268.131714] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 3268.131957] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3268.132195] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3268.132430] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3268.132711] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3268.132933] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3268.133189] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3268.133426] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3268.133661] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3268.133938] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3268.134204] [ 3268.134322] [ 3268.134442] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3268.134723] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3268.134984] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3268.135194] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3268.135394] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3268.135660] ? up_read (kernel/locking/rwsem.c:1622) [ 3268.135836] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3268.136079] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3268.136342] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3268.136604] __tcf_action_put (net/sched/act_api.c:377) [ 3268.136814] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3268.137029] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3268.137238] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3268.137517] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3268.137773] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3268.138020] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3268.138232] tcf_chain_flush (net/sched/cls_api.c:737) [ 3268.138438] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3268.138645] tcf_block_put (net/sched/cls_api.c:1543) [ 3268.138839] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3268.139080] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3268.139288] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3268.139482] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3268.139771] default_device_exit_batch (net/core/dev.c:11629) [ 3268.140027] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3268.140221] ? process_one_work (kernel/workqueue.c:2606) [ 3268.140449] process_one_work (kernel/workqueue.c:2633) [ 3268.140664] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3268.140866] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3268.141098] kthread (kernel/kthread.c:388) [ 3268.141271] ? __pfx_kthread (kernel/kthread.c:341) [ 3268.141474] ret_from_fork (arch/x86/kernel/process.c:147) [ 3268.141670] ? __pfx_kthread (kernel/kthread.c:341) [ 3268.141872] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3322.941874] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 3279533 jiffies s: 6353 root: 0x8/. | [ 3322.942491] rcu: blocking rcu_node structures (internal RCU debug): | [ 3322.942824] Sending NMI from CPU 1 to CPUs 3: | [ 3322.943072] NMI backtrace for cpu 3 [ 3322.943079] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3322.943081] Workqueue: netns cleanup_net [ 3322.943089] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3322.943096] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3322.943098] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3322.943099] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3322.943100] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3322.943101] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3322.943102] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3322.943103] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3322.943106] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3322.943107] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3322.943107] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3322.943108] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3322.943109] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3322.943110] PKRU: 55555554 [ 3322.943110] Call Trace: [ 3322.943113] [ 3322.943115] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 3322.943120] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 3322.943125] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 3322.943128] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 3322.943132] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 3322.943134] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 3322.943138] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3322.943140] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3322.943142] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3322.943144] [ 3322.943144] [ 3322.943145] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3322.943150] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3322.943154] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3322.943159] ? up_read (kernel/locking/rwsem.c:1622) [ 3322.943162] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3322.943166] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3322.943168] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3322.943170] __tcf_action_put (net/sched/act_api.c:377) [ 3322.943176] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3322.943178] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3322.943180] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3322.943187] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3322.943192] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3322.943198] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3322.943201] tcf_chain_flush (net/sched/cls_api.c:737) [ 3322.943203] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3322.943206] tcf_block_put (net/sched/cls_api.c:1543) [ 3322.943210] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3322.943213] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3322.943217] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3322.943220] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3322.943224] default_device_exit_batch (net/core/dev.c:11629) [ 3322.943227] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3322.943229] ? process_one_work (kernel/workqueue.c:2606) [ 3322.943233] process_one_work (kernel/workqueue.c:2633) [ 3322.943236] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3322.943239] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3322.943241] kthread (kernel/kthread.c:388) [ 3322.943244] ? __pfx_kthread (kernel/kthread.c:341) [ 3322.943246] ret_from_fork (arch/x86/kernel/process.c:147) [ 3322.943250] ? __pfx_kthread (kernel/kthread.c:341) [ 3322.943251] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3322.943257] | [ 3346.140864] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3346.141193] rcu: 3-....: (3301489 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=808845 | [ 3346.141718] rcu: (t=3302766 jiffies g=31081 q=2291 ncpus=4) [ 3346.142468] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3346.143066] Workqueue: netns cleanup_net [ 3346.143282] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3346.143579] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3346.144531] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3346.144808] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3346.145179] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3346.145551] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3346.145924] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3346.146304] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3346.146676] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3346.147100] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3346.147402] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3346.147774] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3346.148146] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3346.148518] PKRU: 55555554 [ 3346.148664] Call Trace: [ 3346.148800] [ 3346.148916] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3346.149156] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3346.149400] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 3346.149667] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 3346.149908] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3346.150149] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3346.150383] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3346.150665] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3346.150882] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3346.151137] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3346.151376] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3346.151605] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3346.151884] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3346.152152] [ 3346.152270] [ 3346.152388] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3346.152671] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3346.152931] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3346.153140] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3346.153341] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3346.153606] ? up_read (kernel/locking/rwsem.c:1622) [ 3346.153785] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3346.154027] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3346.154290] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3346.154545] __tcf_action_put (net/sched/act_api.c:377) [ 3346.154756] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3346.154970] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3346.155176] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3346.155456] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3346.155709] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3346.155955] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3346.156170] tcf_chain_flush (net/sched/cls_api.c:737) [ 3346.156379] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3346.156587] tcf_block_put (net/sched/cls_api.c:1543) [ 3346.156782] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3346.157021] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3346.157229] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3346.157425] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3346.157711] default_device_exit_batch (net/core/dev.c:11629) [ 3346.157969] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3346.158165] ? process_one_work (kernel/workqueue.c:2606) [ 3346.158390] process_one_work (kernel/workqueue.c:2633) [ 3346.158608] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3346.158810] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3346.159038] kthread (kernel/kthread.c:388) [ 3346.159212] ? __pfx_kthread (kernel/kthread.c:341) [ 3346.159413] ret_from_fork (arch/x86/kernel/process.c:147) [ 3346.159609] ? __pfx_kthread (kernel/kthread.c:341) [ 3346.159809] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3421.246874] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 3377838 jiffies s: 6353 root: 0x8/. | [ 3421.247480] rcu: blocking rcu_node structures (internal RCU debug): | [ 3421.247813] Sending NMI from CPU 1 to CPUs 3: | [ 3421.248060] NMI backtrace for cpu 3 [ 3421.248065] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3421.248067] Workqueue: netns cleanup_net [ 3421.248072] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3421.248077] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3421.248078] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3421.248080] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3421.248081] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3421.248082] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3421.248083] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3421.248083] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3421.248086] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3421.248087] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3421.248088] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3421.248089] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3421.248089] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3421.248090] PKRU: 55555554 [ 3421.248091] Call Trace: [ 3421.248093] [ 3421.248095] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 3421.248100] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 3421.248104] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 3421.248107] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 3421.248111] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 3421.248113] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 3421.248117] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3421.248119] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3421.248121] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3421.248123] [ 3421.248123] [ 3421.248124] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3421.248128] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3421.248132] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3421.248136] ? up_read (kernel/locking/rwsem.c:1622) [ 3421.248139] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3421.248143] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3421.248145] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3421.248147] __tcf_action_put (net/sched/act_api.c:377) [ 3421.248151] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3421.248153] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3421.248156] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3421.248162] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3421.248168] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3421.248173] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3421.248175] tcf_chain_flush (net/sched/cls_api.c:737) [ 3421.248178] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3421.248181] tcf_block_put (net/sched/cls_api.c:1543) [ 3421.248184] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3421.248189] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3421.248193] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3421.248195] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3421.248199] default_device_exit_batch (net/core/dev.c:11629) [ 3421.248202] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3421.248204] ? process_one_work (kernel/workqueue.c:2606) [ 3421.248207] process_one_work (kernel/workqueue.c:2633) [ 3421.248210] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3421.248212] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3421.248214] kthread (kernel/kthread.c:388) [ 3421.248216] ? __pfx_kthread (kernel/kthread.c:341) [ 3421.248217] ret_from_fork (arch/x86/kernel/process.c:147) [ 3421.248220] ? __pfx_kthread (kernel/kthread.c:341) [ 3421.248221] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3421.248227] | [ 3424.158864] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3424.159194] rcu: 3-....: (3379474 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=828087 | [ 3424.159719] rcu: (t=3380784 jiffies g=31081 q=2291 ncpus=4) [ 3424.160468] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3424.161061] Workqueue: netns cleanup_net [ 3424.161273] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3424.161564] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3424.162525] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3424.162799] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3424.163174] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3424.163547] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3424.163924] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3424.164299] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3424.164674] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3424.165096] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3424.165397] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3424.165776] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3424.166149] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3424.166521] PKRU: 55555554 [ 3424.166668] Call Trace: [ 3424.166803] [ 3424.166919] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3424.167155] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3424.167398] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 3424.167659] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 3424.167900] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3424.168141] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3424.168372] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3424.168655] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3424.168872] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3424.169127] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3424.169366] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3424.169594] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3424.169870] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3424.170135] [ 3424.170252] [ 3424.170369] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3424.170652] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3424.170912] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3424.171120] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3424.171320] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3424.171583] ? up_read (kernel/locking/rwsem.c:1622) [ 3424.171760] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3424.171998] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3424.172261] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3424.172516] __tcf_action_put (net/sched/act_api.c:377) [ 3424.172724] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3424.172939] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3424.173144] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3424.173424] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3424.173678] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3424.173924] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3424.174135] tcf_chain_flush (net/sched/cls_api.c:737) [ 3424.174343] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3424.174551] tcf_block_put (net/sched/cls_api.c:1543) [ 3424.174746] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3424.174986] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3424.175192] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3424.175386] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3424.175673] default_device_exit_batch (net/core/dev.c:11629) [ 3424.175933] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3424.176128] ? process_one_work (kernel/workqueue.c:2606) [ 3424.176350] process_one_work (kernel/workqueue.c:2633) [ 3424.176566] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3424.176769] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3424.176998] kthread (kernel/kthread.c:388) [ 3424.177170] ? __pfx_kthread (kernel/kthread.c:341) [ 3424.177372] ret_from_fork (arch/x86/kernel/process.c:147) [ 3424.177566] ? __pfx_kthread (kernel/kthread.c:341) [ 3424.177768] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3424.177980] | [ 3502.176876] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3502.177219] rcu: 3-....: (3457474 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=846726 | [ 3502.177746] rcu: (t=3458802 jiffies g=31081 q=2291 ncpus=4) [ 3502.178499] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3502.179093] Workqueue: netns cleanup_net [ 3502.179313] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3502.179606] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3502.180567] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3502.180843] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3502.181217] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3502.181589] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3502.181962] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3502.182336] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3502.182711] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3502.183133] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3502.183437] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3502.183813] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3502.184186] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3502.184557] PKRU: 55555554 [ 3502.184707] Call Trace: [ 3502.184847] [ 3502.184967] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3502.185205] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3502.185459] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 3502.185722] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 3502.185963] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3502.186203] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3502.186435] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3502.186718] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3502.186935] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3502.187190] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3502.187430] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3502.187658] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3502.187935] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3502.188200] [ 3502.188317] [ 3502.188436] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3502.188719] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3502.188979] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3502.189188] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3502.189388] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3502.189653] ? up_read (kernel/locking/rwsem.c:1622) [ 3502.189831] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3502.190069] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3502.190331] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3502.190587] __tcf_action_put (net/sched/act_api.c:377) [ 3502.190796] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3502.191011] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3502.191217] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3502.191496] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3502.191751] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3502.191996] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3502.192223] tcf_chain_flush (net/sched/cls_api.c:737) [ 3502.192430] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3502.192639] tcf_block_put (net/sched/cls_api.c:1543) [ 3502.192835] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3502.193075] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3502.193282] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3502.193478] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3502.193769] default_device_exit_batch (net/core/dev.c:11629) [ 3502.194027] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3502.194223] ? process_one_work (kernel/workqueue.c:2606) [ 3502.194448] process_one_work (kernel/workqueue.c:2633) [ 3502.194664] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3502.194869] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3502.195099] kthread (kernel/kthread.c:388) [ 3502.195275] ? __pfx_kthread (kernel/kthread.c:341) [ 3502.195483] ret_from_fork (arch/x86/kernel/process.c:147) [ 3502.195681] ? __pfx_kthread (kernel/kthread.c:341) [ 3502.195884] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3519.549879] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 3476141 jiffies s: 6353 root: 0x8/. | [ 3519.550496] rcu: blocking rcu_node structures (internal RCU debug): | [ 3519.550831] Sending NMI from CPU 1 to CPUs 3: | [ 3519.551086] NMI backtrace for cpu 3 [ 3519.551092] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3519.551094] Workqueue: netns cleanup_net [ 3519.551100] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3519.551105] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3519.551106] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3519.551108] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3519.551109] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3519.551110] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3519.551111] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3519.551112] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3519.551114] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3519.551115] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3519.551116] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3519.551117] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3519.551118] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3519.551119] PKRU: 55555554 [ 3519.551119] Call Trace: [ 3519.551121] [ 3519.551123] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 3519.551127] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 3519.551131] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 3519.551134] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 3519.551137] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 3519.551139] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 3519.551142] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3519.551144] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3519.551147] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3519.551149] [ 3519.551149] [ 3519.551150] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3519.551154] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3519.551157] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3519.551161] ? up_read (kernel/locking/rwsem.c:1622) [ 3519.551164] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3519.551168] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3519.551169] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3519.551171] __tcf_action_put (net/sched/act_api.c:377) [ 3519.551176] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3519.551178] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3519.551180] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3519.551185] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3519.551191] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3519.551196] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3519.551198] tcf_chain_flush (net/sched/cls_api.c:737) [ 3519.551201] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3519.551204] tcf_block_put (net/sched/cls_api.c:1543) [ 3519.551207] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3519.551212] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3519.551215] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3519.551218] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3519.551221] default_device_exit_batch (net/core/dev.c:11629) [ 3519.551224] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3519.551227] ? process_one_work (kernel/workqueue.c:2606) [ 3519.551230] process_one_work (kernel/workqueue.c:2633) [ 3519.551232] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3519.551235] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3519.551236] kthread (kernel/kthread.c:388) [ 3519.551239] ? __pfx_kthread (kernel/kthread.c:341) [ 3519.551240] ret_from_fork (arch/x86/kernel/process.c:147) [ 3519.551243] ? __pfx_kthread (kernel/kthread.c:341) [ 3519.551244] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3519.551249] | [ 3580.194865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3580.195218] rcu: 3-....: (3535459 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=865328 | [ 3580.195747] rcu: (t=3536820 jiffies g=31081 q=2291 ncpus=4) [ 3580.196515] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3580.197116] Workqueue: netns cleanup_net [ 3580.197336] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3580.197630] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3580.198601] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3580.198882] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3580.199258] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3580.199636] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3580.200017] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3580.200389] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3580.200765] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3580.201195] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3580.201498] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3580.201879] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3580.202253] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3580.202628] PKRU: 55555554 [ 3580.202776] Call Trace: [ 3580.202917] [ 3580.203033] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3580.203272] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3580.203514] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 3580.203779] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 3580.204026] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3580.204268] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3580.204501] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3580.204786] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3580.205019] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3580.205277] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3580.205521] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3580.205752] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3580.206035] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3580.206302] [ 3580.206420] [ 3580.206538] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3580.206824] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3580.207090] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3580.207300] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3580.207501] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3580.207769] ? up_read (kernel/locking/rwsem.c:1622) [ 3580.207954] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3580.208194] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3580.208460] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3580.208716] __tcf_action_put (net/sched/act_api.c:377) [ 3580.208934] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3580.209150] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3580.209357] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3580.209639] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3580.209902] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3580.210149] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3580.210361] tcf_chain_flush (net/sched/cls_api.c:737) [ 3580.210568] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3580.210779] tcf_block_put (net/sched/cls_api.c:1543) [ 3580.210980] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3580.211221] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3580.211430] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3580.211625] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3580.211920] default_device_exit_batch (net/core/dev.c:11629) [ 3580.212178] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3580.212374] ? process_one_work (kernel/workqueue.c:2606) [ 3580.212600] process_one_work (kernel/workqueue.c:2633) [ 3580.212818] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3580.213027] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3580.213256] kthread (kernel/kthread.c:388) [ 3580.213433] ? __pfx_kthread (kernel/kthread.c:341) [ 3580.213634] ret_from_fork (arch/x86/kernel/process.c:147) [ 3580.213830] ? __pfx_kthread (kernel/kthread.c:341) [ 3580.214038] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3617.853874] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 3574445 jiffies s: 6353 root: 0x8/. | [ 3617.854485] rcu: blocking rcu_node structures (internal RCU debug): | [ 3617.854834] Sending NMI from CPU 1 to CPUs 3: | [ 3617.855081] NMI backtrace for cpu 3 [ 3617.855086] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3617.855088] Workqueue: netns cleanup_net [ 3617.855093] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3617.855098] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3617.855100] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3617.855101] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3617.855102] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3617.855103] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3617.855104] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3617.855105] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3617.855108] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3617.855109] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3617.855110] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3617.855111] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3617.855111] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3617.855112] PKRU: 55555554 [ 3617.855113] Call Trace: [ 3617.855115] [ 3617.855117] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 3617.855121] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 3617.855124] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 3617.855126] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 3617.855130] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 3617.855131] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 3617.855135] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3617.855138] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3617.855140] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3617.855142] [ 3617.855142] [ 3617.855143] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3617.855147] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3617.855151] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3617.855154] ? up_read (kernel/locking/rwsem.c:1622) [ 3617.855158] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3617.855162] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3617.855163] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3617.855165] __tcf_action_put (net/sched/act_api.c:377) [ 3617.855170] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3617.855172] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3617.855174] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3617.855180] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3617.855186] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3617.855191] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3617.855193] tcf_chain_flush (net/sched/cls_api.c:737) [ 3617.855196] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3617.855199] tcf_block_put (net/sched/cls_api.c:1543) [ 3617.855203] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3617.855208] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3617.855211] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3617.855213] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3617.855217] default_device_exit_batch (net/core/dev.c:11629) [ 3617.855220] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3617.855222] ? process_one_work (kernel/workqueue.c:2606) [ 3617.855225] process_one_work (kernel/workqueue.c:2633) [ 3617.855228] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3617.855230] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3617.855232] kthread (kernel/kthread.c:388) [ 3617.855234] ? __pfx_kthread (kernel/kthread.c:341) [ 3617.855235] ret_from_fork (arch/x86/kernel/process.c:147) [ 3617.855238] ? __pfx_kthread (kernel/kthread.c:341) [ 3617.855240] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3617.855245] | [ 3658.212864] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3658.213193] rcu: 3-....: (3613444 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=884391 | [ 3658.213716] rcu: (t=3614838 jiffies g=31081 q=2291 ncpus=4) [ 3658.214466] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3658.215080] Workqueue: netns cleanup_net [ 3658.215296] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 3658.215588] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 <85> c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: f3 0f 1e fa endbr64 12: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 17: 41 55 push %r13 19: 41 54 push %r12 1b: 55 push %rbp 1c: 48 89 fd mov %rdi,%rbp 1f: 53 push %rbx 20: 66 90 xchg %ax,%ax 22: ba 01 00 00 00 mov $0x1,%edx 27: 8b 45 00 mov 0x0(%rbp),%eax 2a:* 85 c0 test %eax,%eax <-- trapping instruction 2c: 75 14 jne 0x42 2e: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 33: 85 c0 test %eax,%eax 35: 75 f0 jne 0x27 37: 5b pop %rbx 38: 5d pop %rbp 39: 41 5c pop %r12 3b: 41 5d pop %r13 3d: c3 ret 3e: cc int3 3f: cc int3 Code starting with the faulting instruction =========================================== 0: 85 c0 test %eax,%eax 2: 75 14 jne 0x18 4: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 9: 85 c0 test %eax,%eax b: 75 f0 jne 0xfffffffffffffffd d: 5b pop %rbx e: 5d pop %rbp f: 41 5c pop %r12 11: 41 5d pop %r13 13: c3 ret 14: cc int3 15: cc int3 [ 3658.216546] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3658.216821] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3658.217194] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3658.217567] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3658.217940] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3658.218312] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3658.218686] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3658.219108] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3658.219409] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3658.219783] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3658.220156] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3658.220529] PKRU: 55555554 [ 3658.220675] Call Trace: [ 3658.220811] [ 3658.220928] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3658.221164] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3658.221407] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 3658.221670] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 3658.221909] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3658.222150] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3658.222382] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3658.222665] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3658.222883] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3658.223138] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3658.223376] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3658.223605] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3658.223881] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3658.224148] [ 3658.224264] [ 3658.224382] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3658.224673] ? queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 3658.224936] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3658.225145] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3658.225345] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3658.225611] ? up_read (kernel/locking/rwsem.c:1622) [ 3658.225789] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3658.226028] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3658.226292] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3658.226548] __tcf_action_put (net/sched/act_api.c:377) [ 3658.226758] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3658.226975] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3658.227182] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3658.227464] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3658.227719] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3658.227965] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3658.228177] tcf_chain_flush (net/sched/cls_api.c:737) [ 3658.228384] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3658.228593] tcf_block_put (net/sched/cls_api.c:1543) [ 3658.228789] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3658.229031] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3658.229237] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3658.229432] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3658.229721] default_device_exit_batch (net/core/dev.c:11629) [ 3658.229978] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3658.230173] ? process_one_work (kernel/workqueue.c:2606) [ 3658.230397] process_one_work (kernel/workqueue.c:2633) [ 3658.230613] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3658.230817] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3658.231047] kthread (kernel/kthread.c:388) [ 3658.231222] ? __pfx_kthread (kernel/kthread.c:341) [ 3658.231424] ret_from_fork (arch/x86/kernel/process.c:147) [ 3658.231621] ? __pfx_kthread (kernel/kthread.c:341) [ 3658.231822] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3716.157875] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 3672749 jiffies s: 6353 root: 0x8/. | [ 3716.158481] rcu: blocking rcu_node structures (internal RCU debug): | [ 3716.158817] Sending NMI from CPU 1 to CPUs 3: | [ 3716.159066] NMI backtrace for cpu 3 [ 3716.159079] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3716.159083] Workqueue: netns cleanup_net [ 3716.159092] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3716.159098] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3716.159099] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3716.159100] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3716.159102] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3716.159103] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3716.159104] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3716.159105] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3716.159107] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3716.159108] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3716.159109] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3716.159110] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3716.159111] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3716.159112] PKRU: 55555554 [ 3716.159112] Call Trace: [ 3716.159116] [ 3716.159125] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 3716.159129] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 3716.159132] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 3716.159135] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 3716.159138] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 3716.159140] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 3716.159144] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3716.159146] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3716.159148] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3716.159150] [ 3716.159151] [ 3716.159151] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3716.159156] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3716.159159] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3716.159163] ? up_read (kernel/locking/rwsem.c:1622) [ 3716.159166] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3716.159170] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3716.159171] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3716.159173] __tcf_action_put (net/sched/act_api.c:377) [ 3716.159180] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3716.159182] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3716.159184] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3716.159190] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3716.159196] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3716.159201] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3716.159203] tcf_chain_flush (net/sched/cls_api.c:737) [ 3716.159206] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3716.159209] tcf_block_put (net/sched/cls_api.c:1543) [ 3716.159213] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3716.159218] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3716.159221] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3716.159223] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3716.159227] default_device_exit_batch (net/core/dev.c:11629) [ 3716.159229] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3716.159232] ? process_one_work (kernel/workqueue.c:2606) [ 3716.159235] process_one_work (kernel/workqueue.c:2633) [ 3716.159238] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3716.159240] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3716.159242] kthread (kernel/kthread.c:388) [ 3716.159245] ? __pfx_kthread (kernel/kthread.c:341) [ 3716.159246] ret_from_fork (arch/x86/kernel/process.c:147) [ 3716.159249] ? __pfx_kthread (kernel/kthread.c:341) [ 3716.159251] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3716.159255] | [ 3736.230864] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3736.231194] rcu: 3-....: (3691429 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=903673 | [ 3736.231722] rcu: (t=3692856 jiffies g=31081 q=2291 ncpus=4) [ 3736.232473] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3736.233066] Workqueue: netns cleanup_net [ 3736.233280] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3736.233574] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3736.234542] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3736.234820] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3736.235194] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3736.235569] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3736.235943] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3736.236317] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3736.236691] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3736.237116] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3736.237420] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3736.237792] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3736.238166] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3736.238539] PKRU: 55555554 [ 3736.238685] Call Trace: [ 3736.238820] [ 3736.238937] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3736.239174] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3736.239417] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 3736.239681] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 3736.239920] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3736.240160] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3736.240393] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3736.240675] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3736.240892] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3736.241148] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3736.241387] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3736.241616] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3736.241895] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3736.242160] [ 3736.242279] [ 3736.242396] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3736.242679] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3736.242939] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3736.243148] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3736.243347] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3736.243613] ? up_read (kernel/locking/rwsem.c:1622) [ 3736.243796] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3736.244037] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3736.244309] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3736.244564] __tcf_action_put (net/sched/act_api.c:377) [ 3736.244775] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3736.244988] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3736.245195] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3736.245477] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3736.245731] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3736.245984] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3736.246195] tcf_chain_flush (net/sched/cls_api.c:737) [ 3736.246404] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3736.246613] tcf_block_put (net/sched/cls_api.c:1543) [ 3736.246809] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3736.247049] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3736.247259] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3736.247455] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3736.247741] default_device_exit_batch (net/core/dev.c:11629) [ 3736.247999] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3736.248193] ? process_one_work (kernel/workqueue.c:2606) [ 3736.248417] process_one_work (kernel/workqueue.c:2633) [ 3736.248637] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3736.248839] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3736.249067] kthread (kernel/kthread.c:388) [ 3736.249243] ? __pfx_kthread (kernel/kthread.c:341) [ 3736.249444] ret_from_fork (arch/x86/kernel/process.c:147) [ 3736.249643] ? __pfx_kthread (kernel/kthread.c:341) [ 3736.249844] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3736.250061] | [ 3814.248865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3814.249205] rcu: 3-....: (3769429 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=922931 | [ 3814.249731] rcu: (t=3770874 jiffies g=31081 q=2291 ncpus=4) [ 3814.250485] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3814.251079] Workqueue: netns cleanup_net [ 3814.251294] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3814.251586] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3814.252547] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3814.252824] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3814.253201] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3814.253578] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3814.253963] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3814.254336] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3814.254711] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3814.255134] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3814.255437] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3814.255813] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3814.256186] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3814.256559] PKRU: 55555554 [ 3814.256707] Call Trace: [ 3814.256842] [ 3814.256959] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3814.257194] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3814.257439] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 3814.257702] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 3814.257944] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3814.258183] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3814.258415] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3814.258696] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3814.258915] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3814.259171] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3814.259410] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3814.259640] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3814.259916] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3814.260182] [ 3814.260300] [ 3814.260418] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3814.260700] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3814.260961] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3814.261173] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3814.261372] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3814.261641] ? up_read (kernel/locking/rwsem.c:1622) [ 3814.261819] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3814.262060] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3814.262323] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3814.262579] __tcf_action_put (net/sched/act_api.c:377) [ 3814.262787] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3814.263004] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3814.263211] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3814.263491] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3814.263758] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3814.264005] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3814.264217] tcf_chain_flush (net/sched/cls_api.c:737) [ 3814.264423] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3814.264629] tcf_block_put (net/sched/cls_api.c:1543) [ 3814.264824] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3814.265066] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3814.265274] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3814.265467] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3814.265755] default_device_exit_batch (net/core/dev.c:11629) [ 3814.266011] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3814.266206] ? process_one_work (kernel/workqueue.c:2606) [ 3814.266430] process_one_work (kernel/workqueue.c:2633) [ 3814.266644] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3814.266847] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3814.267077] kthread (kernel/kthread.c:388) [ 3814.267251] ? __pfx_kthread (kernel/kthread.c:341) [ 3814.267451] ret_from_fork (arch/x86/kernel/process.c:147) [ 3814.267646] ? __pfx_kthread (kernel/kthread.c:341) [ 3814.267847] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3814.462875] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 3771054 jiffies s: 6353 root: 0x8/. | [ 3814.463489] rcu: blocking rcu_node structures (internal RCU debug): | [ 3814.463829] Sending NMI from CPU 1 to CPUs 3: | [ 3814.464087] NMI backtrace for cpu 3 [ 3814.464092] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3814.464094] Workqueue: netns cleanup_net [ 3814.464096] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3814.464099] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3814.464100] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3814.464101] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3814.464102] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3814.464103] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3814.464104] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3814.464104] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3814.464107] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3814.464108] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3814.464108] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3814.464109] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3814.464110] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3814.464110] PKRU: 55555554 [ 3814.464111] Call Trace: [ 3814.464112] [ 3814.464113] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 3814.464116] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 3814.464119] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 3814.464121] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 3814.464123] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 3814.464125] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 3814.464128] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3814.464130] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3814.464132] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3814.464134] [ 3814.464134] [ 3814.464135] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3814.464137] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3814.464140] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3814.464142] ? up_read (kernel/locking/rwsem.c:1622) [ 3814.464144] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3814.464146] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3814.464148] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3814.464150] __tcf_action_put (net/sched/act_api.c:377) [ 3814.464152] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3814.464153] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3814.464155] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3814.464160] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3814.464165] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3814.464170] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3814.464172] tcf_chain_flush (net/sched/cls_api.c:737) [ 3814.464174] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3814.464177] tcf_block_put (net/sched/cls_api.c:1543) [ 3814.464180] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3814.464183] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3814.464185] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3814.464187] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3814.464190] default_device_exit_batch (net/core/dev.c:11629) [ 3814.464192] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3814.464194] ? process_one_work (kernel/workqueue.c:2606) [ 3814.464196] process_one_work (kernel/workqueue.c:2633) [ 3814.464199] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3814.464201] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3814.464202] kthread (kernel/kthread.c:388) [ 3814.464204] ? __pfx_kthread (kernel/kthread.c:341) [ 3814.464205] ret_from_fork (arch/x86/kernel/process.c:147) [ 3814.464207] ? __pfx_kthread (kernel/kthread.c:341) [ 3814.464208] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3814.464212] | [ 3892.266865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3892.267210] rcu: 3-....: (3847414 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=942092 | [ 3892.267736] rcu: (t=3848892 jiffies g=31081 q=2292 ncpus=4) [ 3892.268494] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3892.269086] Workqueue: netns cleanup_net [ 3892.269300] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3892.269595] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3892.270553] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3892.270831] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3892.271204] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3892.271579] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3892.271951] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3892.272324] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3892.272697] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3892.273123] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3892.273434] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3892.273806] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3892.274182] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3892.274557] PKRU: 55555554 [ 3892.274704] Call Trace: [ 3892.274840] [ 3892.274958] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3892.275197] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3892.275441] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 3892.275706] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 3892.275946] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3892.276187] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3892.276423] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3892.276705] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3892.276925] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3892.277181] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3892.277421] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3892.277651] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3892.277930] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3892.278196] [ 3892.278316] [ 3892.278433] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3892.278717] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3892.278978] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3892.279187] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3892.279387] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3892.279652] ? up_read (kernel/locking/rwsem.c:1622) [ 3892.279831] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3892.280074] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3892.280338] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3892.280597] __tcf_action_put (net/sched/act_api.c:377) [ 3892.280806] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3892.281022] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3892.281229] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3892.281511] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3892.281764] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3892.282013] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3892.282225] tcf_chain_flush (net/sched/cls_api.c:737) [ 3892.282434] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3892.282642] tcf_block_put (net/sched/cls_api.c:1543) [ 3892.282838] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3892.283079] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3892.283289] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3892.283500] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3892.283785] default_device_exit_batch (net/core/dev.c:11629) [ 3892.284044] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3892.284240] ? process_one_work (kernel/workqueue.c:2606) [ 3892.284464] process_one_work (kernel/workqueue.c:2633) [ 3892.284682] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3892.284885] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3892.285113] kthread (kernel/kthread.c:388) [ 3892.285288] ? __pfx_kthread (kernel/kthread.c:341) [ 3892.285489] ret_from_fork (arch/x86/kernel/process.c:147) [ 3892.285683] ? __pfx_kthread (kernel/kthread.c:341) [ 3892.285884] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3912.765878] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 3869357 jiffies s: 6353 root: 0x8/. | [ 3912.766513] rcu: blocking rcu_node structures (internal RCU debug): | [ 3912.766844] Sending NMI from CPU 1 to CPUs 3: | [ 3912.767092] NMI backtrace for cpu 3 [ 3912.767099] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3912.767101] Workqueue: netns cleanup_net [ 3912.767107] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3912.767112] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3912.767114] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3912.767115] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3912.767116] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3912.767117] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3912.767118] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3912.767119] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3912.767122] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3912.767123] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3912.767124] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3912.767125] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3912.767125] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3912.767126] PKRU: 55555554 [ 3912.767127] Call Trace: [ 3912.767128] [ 3912.767131] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 3912.767134] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 3912.767138] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 3912.767141] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 3912.767144] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 3912.767145] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 3912.767149] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3912.767151] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3912.767153] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3912.767156] [ 3912.767156] [ 3912.767157] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3912.767161] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3912.767164] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3912.767168] ? up_read (kernel/locking/rwsem.c:1622) [ 3912.767171] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3912.767175] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3912.767177] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3912.767179] __tcf_action_put (net/sched/act_api.c:377) [ 3912.767184] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3912.767186] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3912.767188] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3912.767194] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3912.767199] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3912.767205] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3912.767208] tcf_chain_flush (net/sched/cls_api.c:737) [ 3912.767210] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3912.767213] tcf_block_put (net/sched/cls_api.c:1543) [ 3912.767216] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3912.767221] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3912.767225] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3912.767227] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3912.767230] default_device_exit_batch (net/core/dev.c:11629) [ 3912.767233] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3912.767236] ? process_one_work (kernel/workqueue.c:2606) [ 3912.767239] process_one_work (kernel/workqueue.c:2633) [ 3912.767241] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3912.767244] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3912.767245] kthread (kernel/kthread.c:388) [ 3912.767247] ? __pfx_kthread (kernel/kthread.c:341) [ 3912.767249] ret_from_fork (arch/x86/kernel/process.c:147) [ 3912.767252] ? __pfx_kthread (kernel/kthread.c:341) [ 3912.767253] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 3912.767257] | [ 3970.284866] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 3970.285208] rcu: 3-....: (3925399 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=961305 | [ 3970.285730] rcu: (t=3926910 jiffies g=31081 q=2292 ncpus=4) [ 3970.286479] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 3970.287072] Workqueue: netns cleanup_net [ 3970.287288] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3970.287581] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 3970.288543] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 3970.288819] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 3970.289194] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 3970.289565] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 3970.289940] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 3970.290314] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 3970.290689] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 3970.291114] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 3970.291418] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 3970.291793] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 3970.292166] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 3970.292540] PKRU: 55555554 [ 3970.292688] Call Trace: [ 3970.292826] [ 3970.292959] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 3970.293212] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 3970.293457] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 3970.293721] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 3970.293961] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 3970.294200] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 3970.294434] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 3970.294715] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 3970.294937] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 3970.295192] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 3970.295430] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 3970.295661] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 3970.295938] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 3970.296202] [ 3970.296324] [ 3970.296443] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 3970.296724] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 3970.296984] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 3970.297193] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 3970.297391] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3970.297656] ? up_read (kernel/locking/rwsem.c:1622) [ 3970.297836] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 3970.298078] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3970.298341] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 3970.298597] __tcf_action_put (net/sched/act_api.c:377) [ 3970.298805] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 3970.299020] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 3970.299227] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 3970.299508] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 3970.299763] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 3970.300009] tcf_proto_destroy (net/sched/cls_api.c:418) [ 3970.300223] tcf_chain_flush (net/sched/cls_api.c:737) [ 3970.300429] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 3970.300636] tcf_block_put (net/sched/cls_api.c:1543) [ 3970.300832] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 3970.301074] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 3970.301282] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 3970.301476] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 3970.301762] default_device_exit_batch (net/core/dev.c:11629) [ 3970.302019] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 3970.302213] ? process_one_work (kernel/workqueue.c:2606) [ 3970.302439] process_one_work (kernel/workqueue.c:2633) [ 3970.302655] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 3970.302857] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 3970.303119] kthread (kernel/kthread.c:388) [ 3970.303291] ? __pfx_kthread (kernel/kthread.c:341) [ 3970.303496] ret_from_fork (arch/x86/kernel/process.c:147) [ 3970.303691] ? __pfx_kthread (kernel/kthread.c:341) [ 3970.303894] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4011.069893] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 3967661 jiffies s: 6353 root: 0x8/. | [ 4011.070511] rcu: blocking rcu_node structures (internal RCU debug): | [ 4011.070844] Sending NMI from CPU 1 to CPUs 3: | [ 4011.071108] NMI backtrace for cpu 3 [ 4011.071117] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4011.071120] Workqueue: netns cleanup_net [ 4011.071128] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4011.071134] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4011.071135] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 4011.071137] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 4011.071138] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 4011.071139] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 4011.071140] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 4011.071140] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 4011.071143] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 4011.071145] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4011.071145] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 4011.071146] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4011.071147] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4011.071148] PKRU: 55555554 [ 4011.071148] Call Trace: [ 4011.071155] [ 4011.071160] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 4011.071163] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 4011.071167] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 4011.071169] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 4011.071172] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 4011.071174] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 4011.071178] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4011.071180] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4011.071182] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4011.071184] [ 4011.071184] [ 4011.071185] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4011.071190] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4011.071194] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4011.071198] ? up_read (kernel/locking/rwsem.c:1622) [ 4011.071201] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4011.071205] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4011.071206] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4011.071208] __tcf_action_put (net/sched/act_api.c:377) [ 4011.071214] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4011.071216] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4011.071218] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4011.071225] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4011.071230] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4011.071235] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4011.071238] tcf_chain_flush (net/sched/cls_api.c:737) [ 4011.071240] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4011.071243] tcf_block_put (net/sched/cls_api.c:1543) [ 4011.071246] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4011.071251] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4011.071254] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4011.071257] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 4011.071261] default_device_exit_batch (net/core/dev.c:11629) [ 4011.071263] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4011.071266] ? process_one_work (kernel/workqueue.c:2606) [ 4011.071269] process_one_work (kernel/workqueue.c:2633) [ 4011.071272] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4011.071274] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4011.071276] kthread (kernel/kthread.c:388) [ 4011.071278] ? __pfx_kthread (kernel/kthread.c:341) [ 4011.071279] ret_from_fork (arch/x86/kernel/process.c:147) [ 4011.071282] ? __pfx_kthread (kernel/kthread.c:341) [ 4011.071284] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4011.071289] | [ 4048.302865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4048.303205] rcu: 3-....: (4003384 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=980343 | [ 4048.303732] rcu: (t=4004928 jiffies g=31081 q=2292 ncpus=4) [ 4048.304483] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4048.305077] Workqueue: netns cleanup_net [ 4048.305291] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 4048.305585] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 <85> c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc All code ======== 0: 90 nop 1: 90 nop 2: 90 nop 3: 90 nop 4: 90 nop 5: 90 nop 6: 90 nop 7: 90 nop 8: 90 nop 9: 90 nop a: 90 nop b: 90 nop c: 90 nop d: 90 nop e: f3 0f 1e fa endbr64 12: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 17: 41 55 push %r13 19: 41 54 push %r12 1b: 55 push %rbp 1c: 48 89 fd mov %rdi,%rbp 1f: 53 push %rbx 20: 66 90 xchg %ax,%ax 22: ba 01 00 00 00 mov $0x1,%edx 27: 8b 45 00 mov 0x0(%rbp),%eax 2a:* 85 c0 test %eax,%eax <-- trapping instruction 2c: 75 14 jne 0x42 2e: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 33: 85 c0 test %eax,%eax 35: 75 f0 jne 0x27 37: 5b pop %rbx 38: 5d pop %rbp 39: 41 5c pop %r12 3b: 41 5d pop %r13 3d: c3 ret 3e: cc int3 3f: cc int3 Code starting with the faulting instruction =========================================== 0: 85 c0 test %eax,%eax 2: 75 14 jne 0x18 4: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 9: 85 c0 test %eax,%eax b: 75 f0 jne 0xfffffffffffffffd d: 5b pop %rbx e: 5d pop %rbp f: 41 5c pop %r12 11: 41 5d pop %r13 13: c3 ret 14: cc int3 15: cc int3 [ 4048.306544] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 4048.306822] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 4048.307194] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 4048.307567] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 4048.307941] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 4048.308315] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 4048.308688] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 4048.309111] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4048.309414] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 4048.309787] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4048.310162] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4048.310536] PKRU: 55555554 [ 4048.310683] Call Trace: [ 4048.310819] [ 4048.310936] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4048.311175] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4048.311419] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 4048.311683] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 4048.311923] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4048.312164] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4048.312399] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4048.312686] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4048.312906] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4048.313164] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4048.313404] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4048.313635] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4048.313913] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4048.314179] [ 4048.314298] [ 4048.314416] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4048.314698] ? queued_spin_lock_slowpath (./arch/x86/include/asm/qspinlock.h:98 kernel/locking/qspinlock.c:327) [ 4048.314959] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4048.315168] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4048.315368] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4048.315633] ? up_read (kernel/locking/rwsem.c:1622) [ 4048.315811] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4048.316055] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4048.316318] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4048.316573] __tcf_action_put (net/sched/act_api.c:377) [ 4048.316782] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4048.316997] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4048.317204] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4048.317486] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4048.317740] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4048.317987] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4048.318199] tcf_chain_flush (net/sched/cls_api.c:737) [ 4048.318407] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4048.318614] tcf_block_put (net/sched/cls_api.c:1543) [ 4048.318809] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4048.319051] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4048.319259] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4048.319454] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 4048.319740] default_device_exit_batch (net/core/dev.c:11629) [ 4048.319997] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4048.320193] ? process_one_work (kernel/workqueue.c:2606) [ 4048.320418] process_one_work (kernel/workqueue.c:2633) [ 4048.320635] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4048.320837] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4048.321068] kthread (kernel/kthread.c:388) [ 4048.321241] ? __pfx_kthread (kernel/kthread.c:341) [ 4048.321445] ret_from_fork (arch/x86/kernel/process.c:147) [ 4048.321639] ? __pfx_kthread (kernel/kthread.c:341) [ 4048.321841] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4109.373878] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 4065965 jiffies s: 6353 root: 0x8/. | [ 4109.374534] rcu: blocking rcu_node structures (internal RCU debug): | [ 4109.374889] Sending NMI from CPU 1 to CPUs 3: | [ 4109.375144] NMI backtrace for cpu 3 [ 4109.375159] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4109.375160] Workqueue: netns cleanup_net [ 4109.375169] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4109.375174] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4109.375175] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 4109.375177] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 4109.375178] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 4109.375179] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 4109.375179] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 4109.375180] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 4109.375183] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 4109.375184] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4109.375185] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 4109.375186] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4109.375186] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4109.375187] PKRU: 55555554 [ 4109.375188] Call Trace: [ 4109.375190] [ 4109.375192] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 4109.375196] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 4109.375199] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 4109.375202] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 4109.375205] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 4109.375207] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 4109.375210] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4109.375213] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4109.375215] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4109.375216] [ 4109.375217] [ 4109.375218] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4109.375222] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4109.375225] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4109.375229] ? up_read (kernel/locking/rwsem.c:1622) [ 4109.375232] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4109.375236] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4109.375238] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4109.375240] __tcf_action_put (net/sched/act_api.c:377) [ 4109.375245] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4109.375247] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4109.375249] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4109.375255] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4109.375261] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4109.375266] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4109.375268] tcf_chain_flush (net/sched/cls_api.c:737) [ 4109.375271] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4109.375274] tcf_block_put (net/sched/cls_api.c:1543) [ 4109.375277] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4109.375282] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4109.375285] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4109.375288] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 4109.375291] default_device_exit_batch (net/core/dev.c:11629) [ 4109.375294] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4109.375297] ? process_one_work (kernel/workqueue.c:2606) [ 4109.375301] process_one_work (kernel/workqueue.c:2633) [ 4109.375304] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4109.375306] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4109.375308] kthread (kernel/kthread.c:388) [ 4109.375310] ? __pfx_kthread (kernel/kthread.c:341) [ 4109.375311] ret_from_fork (arch/x86/kernel/process.c:147) [ 4109.375314] ? __pfx_kthread (kernel/kthread.c:341) [ 4109.375315] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4109.375320] | [ 4126.320865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4126.321199] rcu: 3-....: (4081369 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=997839 | [ 4126.321723] rcu: (t=4082946 jiffies g=31081 q=2292 ncpus=4) [ 4126.322476] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4126.323070] Workqueue: netns cleanup_net [ 4126.323286] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4126.323577] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4126.324539] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 4126.324814] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 4126.325189] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 4126.325561] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 4126.325933] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 4126.326306] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 4126.326681] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 4126.327103] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4126.327406] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 4126.327779] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4126.328153] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4126.328524] PKRU: 55555554 [ 4126.328673] Call Trace: [ 4126.328809] [ 4126.328925] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4126.329161] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4126.329405] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 4126.329669] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 4126.329911] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4126.330148] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4126.330383] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4126.330663] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4126.330884] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4126.331140] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4126.331377] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4126.331608] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4126.331886] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4126.332156] [ 4126.332274] [ 4126.332393] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4126.332675] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4126.332938] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4126.333146] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4126.333345] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4126.333610] ? up_read (kernel/locking/rwsem.c:1622) [ 4126.333786] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4126.334027] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4126.334290] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4126.334545] __tcf_action_put (net/sched/act_api.c:377) [ 4126.334754] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4126.334970] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4126.335179] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4126.335459] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4126.335715] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4126.335962] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4126.336174] tcf_chain_flush (net/sched/cls_api.c:737) [ 4126.336380] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4126.336588] tcf_block_put (net/sched/cls_api.c:1543) [ 4126.336783] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4126.337027] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4126.337234] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4126.337429] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 4126.337714] default_device_exit_batch (net/core/dev.c:11629) [ 4126.337970] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4126.338168] ? process_one_work (kernel/workqueue.c:2606) [ 4126.338395] process_one_work (kernel/workqueue.c:2633) [ 4126.338610] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4126.338813] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4126.339045] kthread (kernel/kthread.c:388) [ 4126.339217] ? __pfx_kthread (kernel/kthread.c:341) [ 4126.339420] ret_from_fork (arch/x86/kernel/process.c:147) [ 4126.339615] ? __pfx_kthread (kernel/kthread.c:341) [ 4126.339817] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4126.340031] | [ 4204.338867] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4204.339200] rcu: 3-....: (4159369 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=1015595 | [ 4204.339729] rcu: (t=4160964 jiffies g=31081 q=2292 ncpus=4) [ 4204.340486] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4204.341084] Workqueue: netns cleanup_net [ 4204.341304] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4204.341596] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4204.342562] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 4204.342837] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 4204.343212] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 4204.343585] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 4204.343958] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 4204.344329] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 4204.344708] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 4204.345130] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4204.345432] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 4204.345807] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4204.346181] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4204.346553] PKRU: 55555554 [ 4204.346700] Call Trace: [ 4204.346840] [ 4204.346963] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4204.347202] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4204.347446] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 4204.347710] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 4204.347949] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4204.348190] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4204.348424] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4204.348707] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4204.348925] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4204.349179] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4204.349418] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4204.349650] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4204.349927] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4204.350192] [ 4204.350310] [ 4204.350428] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4204.350710] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4204.350970] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4204.351178] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4204.351377] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4204.351641] ? up_read (kernel/locking/rwsem.c:1622) [ 4204.351825] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4204.352066] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4204.352330] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4204.352586] __tcf_action_put (net/sched/act_api.c:377) [ 4204.352797] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4204.353012] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4204.353221] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4204.353502] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4204.353757] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4204.354002] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4204.354215] tcf_chain_flush (net/sched/cls_api.c:737) [ 4204.354421] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4204.354628] tcf_block_put (net/sched/cls_api.c:1543) [ 4204.354823] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4204.355066] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4204.355274] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4204.355468] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 4204.355753] default_device_exit_batch (net/core/dev.c:11629) [ 4204.356011] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4204.356206] ? process_one_work (kernel/workqueue.c:2606) [ 4204.356432] process_one_work (kernel/workqueue.c:2633) [ 4204.356647] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4204.356849] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4204.357080] kthread (kernel/kthread.c:388) [ 4204.357251] ? __pfx_kthread (kernel/kthread.c:341) [ 4204.357455] ret_from_fork (arch/x86/kernel/process.c:147) [ 4204.357649] ? __pfx_kthread (kernel/kthread.c:341) [ 4204.357851] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4207.678875] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 4164270 jiffies s: 6353 root: 0x8/. | [ 4207.679480] rcu: blocking rcu_node structures (internal RCU debug): | [ 4207.679814] Sending NMI from CPU 1 to CPUs 3: | [ 4207.680069] NMI backtrace for cpu 3 [ 4207.680075] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4207.680077] Workqueue: netns cleanup_net [ 4207.680083] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4207.680088] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4207.680089] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 4207.680091] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 4207.680092] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 4207.680093] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 4207.680094] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 4207.680094] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 4207.680098] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 4207.680099] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4207.680100] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 4207.680101] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4207.680101] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4207.680102] PKRU: 55555554 [ 4207.680103] Call Trace: [ 4207.680105] [ 4207.680107] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 4207.680110] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 4207.680114] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 4207.680116] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 4207.680119] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 4207.680121] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 4207.680124] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4207.680126] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4207.680128] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4207.680130] [ 4207.680131] [ 4207.680132] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4207.680136] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4207.680139] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4207.680142] ? up_read (kernel/locking/rwsem.c:1622) [ 4207.680145] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4207.680149] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4207.680150] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4207.680152] __tcf_action_put (net/sched/act_api.c:377) [ 4207.680156] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4207.680158] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4207.680160] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4207.680165] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4207.680171] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4207.680176] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4207.680178] tcf_chain_flush (net/sched/cls_api.c:737) [ 4207.680180] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4207.680183] tcf_block_put (net/sched/cls_api.c:1543) [ 4207.680187] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4207.680191] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4207.680195] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4207.680197] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 4207.680200] default_device_exit_batch (net/core/dev.c:11629) [ 4207.680203] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4207.680206] ? process_one_work (kernel/workqueue.c:2606) [ 4207.680209] process_one_work (kernel/workqueue.c:2633) [ 4207.680212] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4207.680214] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4207.680216] kthread (kernel/kthread.c:388) [ 4207.680218] ? __pfx_kthread (kernel/kthread.c:341) [ 4207.680219] ret_from_fork (arch/x86/kernel/process.c:147) [ 4207.680222] ? __pfx_kthread (kernel/kthread.c:341) [ 4207.680223] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4207.680228] | [ 4282.357866] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4282.358201] rcu: 3-....: (4237355 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=1033675 | [ 4282.358731] rcu: (t=4238983 jiffies g=31081 q=2292 ncpus=4) [ 4282.359482] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4282.360077] Workqueue: netns cleanup_net [ 4282.360294] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4282.360586] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4282.361568] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 4282.361845] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 4282.362221] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 4282.362594] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 4282.362971] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 4282.363346] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 4282.363722] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 4282.364147] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4282.364452] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 4282.364827] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4282.365201] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4282.365575] PKRU: 55555554 [ 4282.365723] Call Trace: [ 4282.365861] [ 4282.365976] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4282.366214] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4282.366458] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 4282.366723] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 4282.366965] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4282.367204] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4282.367439] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4282.367721] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4282.367941] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4282.368198] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4282.368436] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4282.368668] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4282.368945] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4282.369211] [ 4282.369329] [ 4282.369448] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4282.369730] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4282.369991] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4282.370199] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4282.370399] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4282.370665] ? up_read (kernel/locking/rwsem.c:1622) [ 4282.370843] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4282.371085] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4282.371370] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4282.371626] __tcf_action_put (net/sched/act_api.c:377) [ 4282.371835] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4282.372050] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4282.372259] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4282.372543] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4282.372799] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4282.373045] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4282.373258] tcf_chain_flush (net/sched/cls_api.c:737) [ 4282.373466] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4282.373674] tcf_block_put (net/sched/cls_api.c:1543) [ 4282.373870] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4282.374112] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4282.374319] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4282.374515] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 4282.374801] default_device_exit_batch (net/core/dev.c:11629) [ 4282.375059] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4282.375254] ? process_one_work (kernel/workqueue.c:2606) [ 4282.375481] process_one_work (kernel/workqueue.c:2633) [ 4282.375697] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4282.375900] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4282.376131] kthread (kernel/kthread.c:388) [ 4282.376302] ? __pfx_kthread (kernel/kthread.c:341) [ 4282.376506] ret_from_fork (arch/x86/kernel/process.c:147) [ 4282.376701] ? __pfx_kthread (kernel/kthread.c:341) [ 4282.376914] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4305.981875] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 4262573 jiffies s: 6353 root: 0x8/. | [ 4305.982498] rcu: blocking rcu_node structures (internal RCU debug): | [ 4305.982831] Sending NMI from CPU 1 to CPUs 3: | [ 4305.983084] NMI backtrace for cpu 3 [ 4305.983091] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4305.983093] Workqueue: netns cleanup_net [ 4305.983098] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4305.983104] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4305.983105] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 4305.983106] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 4305.983108] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 4305.983109] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 4305.983109] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 4305.983110] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 4305.983113] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 4305.983114] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4305.983115] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 4305.983116] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4305.983116] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4305.983117] PKRU: 55555554 [ 4305.983118] Call Trace: [ 4305.983120] [ 4305.983121] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 4305.983125] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 4305.983129] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 4305.983131] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 4305.983134] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 4305.983136] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 4305.983140] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4305.983142] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4305.983145] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4305.983147] [ 4305.983147] [ 4305.983148] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4305.983152] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4305.983156] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4305.983160] ? up_read (kernel/locking/rwsem.c:1622) [ 4305.983163] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4305.983167] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4305.983168] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4305.983170] __tcf_action_put (net/sched/act_api.c:377) [ 4305.983175] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4305.983177] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4305.983179] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4305.983185] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4305.983190] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4305.983195] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4305.983198] tcf_chain_flush (net/sched/cls_api.c:737) [ 4305.983201] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4305.983204] tcf_block_put (net/sched/cls_api.c:1543) [ 4305.983207] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4305.983212] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4305.983215] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4305.983218] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 4305.983221] default_device_exit_batch (net/core/dev.c:11629) [ 4305.983224] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4305.983226] ? process_one_work (kernel/workqueue.c:2606) [ 4305.983229] process_one_work (kernel/workqueue.c:2633) [ 4305.983232] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4305.983234] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4305.983236] kthread (kernel/kthread.c:388) [ 4305.983239] ? __pfx_kthread (kernel/kthread.c:341) [ 4305.983240] ret_from_fork (arch/x86/kernel/process.c:147) [ 4305.983243] ? __pfx_kthread (kernel/kthread.c:341) [ 4305.983245] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4305.983249] | [ 4360.375866] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4360.376205] rcu: 3-....: (4315340 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=1051798 | [ 4360.376737] rcu: (t=4317001 jiffies g=31081 q=2292 ncpus=4) [ 4360.377487] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4360.378080] Workqueue: netns cleanup_net [ 4360.378295] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4360.378589] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4360.379548] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 4360.379824] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 4360.380196] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 4360.380568] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 4360.380959] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 4360.381334] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 4360.381711] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 4360.382138] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4360.382446] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 4360.382823] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4360.383199] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4360.383576] PKRU: 55555554 [ 4360.383724] Call Trace: [ 4360.383864] [ 4360.383979] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4360.384218] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4360.384461] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 4360.384724] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 4360.384963] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4360.385202] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4360.385436] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4360.385717] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4360.385936] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4360.386190] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4360.386430] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4360.386660] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4360.386938] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4360.387204] [ 4360.387322] [ 4360.387440] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4360.387721] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4360.387982] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4360.388189] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4360.388388] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4360.388652] ? up_read (kernel/locking/rwsem.c:1622) [ 4360.388831] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4360.389072] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4360.389336] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4360.389591] __tcf_action_put (net/sched/act_api.c:377) [ 4360.389799] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4360.390014] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4360.390220] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4360.390501] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4360.390755] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4360.391014] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4360.391227] tcf_chain_flush (net/sched/cls_api.c:737) [ 4360.391434] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4360.391641] tcf_block_put (net/sched/cls_api.c:1543) [ 4360.391837] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4360.392079] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4360.392286] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4360.392480] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 4360.392765] default_device_exit_batch (net/core/dev.c:11629) [ 4360.393022] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4360.393217] ? process_one_work (kernel/workqueue.c:2606) [ 4360.393441] process_one_work (kernel/workqueue.c:2633) [ 4360.393657] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4360.393859] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4360.394089] kthread (kernel/kthread.c:388) [ 4360.394260] ? __pfx_kthread (kernel/kthread.c:341) [ 4360.394461] ret_from_fork (arch/x86/kernel/process.c:147) [ 4360.394656] ? __pfx_kthread (kernel/kthread.c:341) [ 4360.394858] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4404.286878] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 4360878 jiffies s: 6353 root: 0x8/. | [ 4404.287505] rcu: blocking rcu_node structures (internal RCU debug): | [ 4404.287847] Sending NMI from CPU 1 to CPUs 3: | [ 4404.288095] NMI backtrace for cpu 3 [ 4404.288102] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4404.288103] Workqueue: netns cleanup_net [ 4404.288109] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4404.288114] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4404.288116] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 4404.288117] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 4404.288118] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 4404.288119] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 4404.288120] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 4404.288121] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 4404.288124] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 4404.288125] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4404.288126] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 4404.288127] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4404.288127] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4404.288128] PKRU: 55555554 [ 4404.288129] Call Trace: [ 4404.288131] [ 4404.288133] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 4404.288137] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 4404.288141] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 4404.288144] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 4404.288147] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 4404.288149] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 4404.288152] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4404.288155] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4404.288157] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4404.288159] [ 4404.288159] [ 4404.288160] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4404.288164] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4404.288167] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4404.288171] ? up_read (kernel/locking/rwsem.c:1622) [ 4404.288174] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4404.288178] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4404.288179] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4404.288181] __tcf_action_put (net/sched/act_api.c:377) [ 4404.288186] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4404.288188] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4404.288190] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4404.288196] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4404.288201] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4404.288206] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4404.288209] tcf_chain_flush (net/sched/cls_api.c:737) [ 4404.288212] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4404.288214] tcf_block_put (net/sched/cls_api.c:1543) [ 4404.288218] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4404.288222] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4404.288226] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4404.288228] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 4404.288232] default_device_exit_batch (net/core/dev.c:11629) [ 4404.288234] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4404.288237] ? process_one_work (kernel/workqueue.c:2606) [ 4404.288240] process_one_work (kernel/workqueue.c:2633) [ 4404.288243] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4404.288245] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4404.288247] kthread (kernel/kthread.c:388) [ 4404.288249] ? __pfx_kthread (kernel/kthread.c:341) [ 4404.288251] ret_from_fork (arch/x86/kernel/process.c:147) [ 4404.288253] ? __pfx_kthread (kernel/kthread.c:341) [ 4404.288255] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4404.288259] | [ 4438.394865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4438.395202] rcu: 3-....: (4393326 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=1070157 | [ 4438.395731] rcu: (t=4395020 jiffies g=31081 q=2292 ncpus=4) [ 4438.396494] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4438.397095] Workqueue: netns cleanup_net [ 4438.397312] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4438.397606] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4438.398577] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 4438.398855] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 4438.399234] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 4438.399609] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 4438.399985] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 4438.400363] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 4438.400743] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 4438.401167] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4438.401471] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 4438.401844] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4438.402218] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4438.402588] PKRU: 55555554 [ 4438.402738] Call Trace: [ 4438.402874] [ 4438.402990] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4438.403225] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4438.403469] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 4438.403732] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 4438.403973] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4438.404212] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4438.404446] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4438.404725] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4438.404946] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4438.405201] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4438.405438] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4438.405669] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4438.405946] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4438.406210] [ 4438.406328] [ 4438.406446] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4438.406726] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4438.406986] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4438.407196] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4438.407396] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4438.407662] ? up_read (kernel/locking/rwsem.c:1622) [ 4438.407838] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4438.408080] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4438.408342] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4438.408597] __tcf_action_put (net/sched/act_api.c:377) [ 4438.408805] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4438.409021] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4438.409229] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4438.409508] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4438.409763] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4438.410010] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4438.410223] tcf_chain_flush (net/sched/cls_api.c:737) [ 4438.410429] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4438.410667] tcf_block_put (net/sched/cls_api.c:1543) [ 4438.410863] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4438.411105] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4438.411313] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4438.411507] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 4438.411795] default_device_exit_batch (net/core/dev.c:11629) [ 4438.412053] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4438.412247] ? process_one_work (kernel/workqueue.c:2606) [ 4438.412473] process_one_work (kernel/workqueue.c:2633) [ 4438.412688] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4438.412891] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4438.413123] kthread (kernel/kthread.c:388) [ 4438.413297] ? __pfx_kthread (kernel/kthread.c:341) [ 4438.413499] ret_from_fork (arch/x86/kernel/process.c:147) [ 4438.413694] ? __pfx_kthread (kernel/kthread.c:341) [ 4438.413900] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4502.590885] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 4459182 jiffies s: 6353 root: 0x8/. | [ 4502.591487] rcu: blocking rcu_node structures (internal RCU debug): | [ 4502.591820] Sending NMI from CPU 1 to CPUs 3: | [ 4502.592081] NMI backtrace for cpu 3 [ 4502.592091] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4502.592094] Workqueue: netns cleanup_net [ 4502.592101] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4502.592106] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4502.592108] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 4502.592109] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 4502.592111] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 4502.592112] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 4502.592112] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 4502.592113] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 4502.592116] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 4502.592117] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4502.592118] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 4502.592119] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4502.592119] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4502.592120] PKRU: 55555554 [ 4502.592121] Call Trace: [ 4502.592127] [ 4502.592131] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 4502.592135] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 4502.592139] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 4502.592142] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 4502.592146] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 4502.592147] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 4502.592152] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4502.592154] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4502.592156] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4502.592158] [ 4502.592158] [ 4502.592159] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4502.592163] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4502.592167] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4502.592170] ? up_read (kernel/locking/rwsem.c:1622) [ 4502.592173] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4502.592177] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4502.592179] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4502.592181] __tcf_action_put (net/sched/act_api.c:377) [ 4502.592185] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4502.592187] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4502.592189] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4502.592195] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4502.592201] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4502.592206] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4502.592209] tcf_chain_flush (net/sched/cls_api.c:737) [ 4502.592211] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4502.592214] tcf_block_put (net/sched/cls_api.c:1543) [ 4502.592217] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4502.592222] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4502.592225] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4502.592228] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 4502.592231] default_device_exit_batch (net/core/dev.c:11629) [ 4502.592234] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4502.592237] ? process_one_work (kernel/workqueue.c:2606) [ 4502.592239] process_one_work (kernel/workqueue.c:2633) [ 4502.592242] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4502.592244] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4502.592246] kthread (kernel/kthread.c:388) [ 4502.592248] ? __pfx_kthread (kernel/kthread.c:341) [ 4502.592249] ret_from_fork (arch/x86/kernel/process.c:147) [ 4502.592252] ? __pfx_kthread (kernel/kthread.c:341) [ 4502.592253] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4502.592258] | [ 4516.412866] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4516.413213] rcu: 3-....: (4471311 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=1088414 | [ 4516.413743] rcu: (t=4473038 jiffies g=31081 q=2292 ncpus=4) [ 4516.414497] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4516.415094] Workqueue: netns cleanup_net [ 4516.415311] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4516.415606] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4516.416570] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 4516.416847] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 4516.417223] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 4516.417597] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 4516.417974] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 4516.418348] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 4516.418726] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 4516.419150] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4516.419455] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 4516.419831] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4516.420207] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4516.420579] PKRU: 55555554 [ 4516.420726] Call Trace: [ 4516.420863] [ 4516.420977] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4516.421214] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4516.421458] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 4516.421722] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 4516.421962] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4516.422200] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4516.422434] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4516.422713] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4516.422933] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4516.423187] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4516.423426] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4516.423658] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4516.423934] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4516.424199] [ 4516.424317] [ 4516.424436] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4516.424717] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4516.424976] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4516.425182] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4516.425381] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4516.425646] ? up_read (kernel/locking/rwsem.c:1622) [ 4516.425823] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4516.426063] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4516.426326] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4516.426581] __tcf_action_put (net/sched/act_api.c:377) [ 4516.426789] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4516.427003] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4516.427211] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4516.427492] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4516.427746] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4516.427991] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4516.428203] tcf_chain_flush (net/sched/cls_api.c:737) [ 4516.428409] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4516.428616] tcf_block_put (net/sched/cls_api.c:1543) [ 4516.428811] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4516.429052] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4516.429258] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4516.429452] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 4516.429737] default_device_exit_batch (net/core/dev.c:11629) [ 4516.429994] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4516.430192] ? process_one_work (kernel/workqueue.c:2606) [ 4516.430417] process_one_work (kernel/workqueue.c:2633) [ 4516.430631] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4516.430833] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4516.431063] kthread (kernel/kthread.c:388) [ 4516.431233] ? __pfx_kthread (kernel/kthread.c:341) [ 4516.431435] ret_from_fork (arch/x86/kernel/process.c:147) [ 4516.431630] ? __pfx_kthread (kernel/kthread.c:341) [ 4516.431832] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4516.432046] | [ 4594.431865] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4594.432213] rcu: 3-....: (4549312 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=1106566 | [ 4594.432743] rcu: (t=4551057 jiffies g=31081 q=2292 ncpus=4) [ 4594.433498] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4594.434093] Workqueue: netns cleanup_net [ 4594.434316] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4594.434607] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4594.435571] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 4594.435848] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 4594.436224] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 4594.436599] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 4594.436973] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 4594.437348] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 4594.437724] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 4594.438148] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4594.438456] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 4594.438837] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4594.439215] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4594.439591] PKRU: 55555554 [ 4594.439746] Call Trace: [ 4594.439885] [ 4594.440002] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4594.440240] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4594.440486] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 4594.440753] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 4594.440997] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4594.441239] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4594.441474] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4594.441759] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4594.441984] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4594.442242] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4594.442483] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4594.442716] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4594.442997] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4594.443267] [ 4594.443387] [ 4594.443507] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4594.443791] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4594.444055] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4594.444274] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4594.444475] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4594.444743] ? up_read (kernel/locking/rwsem.c:1622) [ 4594.444922] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4594.445165] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4594.445430] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4594.445688] __tcf_action_put (net/sched/act_api.c:377) [ 4594.445899] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4594.446118] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4594.446327] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4594.446610] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4594.446869] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4594.447119] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4594.447333] tcf_chain_flush (net/sched/cls_api.c:737) [ 4594.447541] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4594.447751] tcf_block_put (net/sched/cls_api.c:1543) [ 4594.447948] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4594.448192] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4594.448401] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4594.448597] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 4594.448890] default_device_exit_batch (net/core/dev.c:11629) [ 4594.449148] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4594.449345] ? process_one_work (kernel/workqueue.c:2606) [ 4594.449573] process_one_work (kernel/workqueue.c:2633) [ 4594.449795] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4594.450001] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4594.450234] kthread (kernel/kthread.c:388) [ 4594.450409] ? __pfx_kthread (kernel/kthread.c:341) [ 4594.450614] ret_from_fork (arch/x86/kernel/process.c:147) [ 4594.450812] ? __pfx_kthread (kernel/kthread.c:341) [ 4594.451016] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4600.893890] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 4557485 jiffies s: 6353 root: 0x8/. | [ 4600.894767] rcu: blocking rcu_node structures (internal RCU debug): | [ 4600.895284] Sending NMI from CPU 1 to CPUs 3: | [ 4600.895633] NMI backtrace for cpu 3 [ 4600.895641] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4600.895642] Workqueue: netns cleanup_net [ 4600.895649] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4600.895654] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4600.895656] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 4600.895657] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 4600.895658] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 4600.895659] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 4600.895660] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 4600.895661] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 4600.895663] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 4600.895664] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4600.895665] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 4600.895666] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4600.895666] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4600.895667] PKRU: 55555554 [ 4600.895667] Call Trace: [ 4600.895669] [ 4600.895671] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 4600.895674] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 4600.895678] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 4600.895680] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 4600.895683] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 4600.895685] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 4600.895688] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4600.895691] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4600.895693] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4600.895695] [ 4600.895695] [ 4600.895696] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4600.895700] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4600.895703] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4600.895706] ? up_read (kernel/locking/rwsem.c:1622) [ 4600.895710] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4600.895713] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4600.895715] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4600.895717] __tcf_action_put (net/sched/act_api.c:377) [ 4600.895721] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4600.895723] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4600.895725] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4600.895731] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4600.895736] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4600.895741] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4600.895744] tcf_chain_flush (net/sched/cls_api.c:737) [ 4600.895746] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4600.895749] tcf_block_put (net/sched/cls_api.c:1543) [ 4600.895752] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4600.895756] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4600.895759] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4600.895761] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 4600.895765] default_device_exit_batch (net/core/dev.c:11629) [ 4600.895768] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4600.895771] ? process_one_work (kernel/workqueue.c:2606) [ 4600.895774] process_one_work (kernel/workqueue.c:2633) [ 4600.895777] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4600.895779] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4600.895781] kthread (kernel/kthread.c:388) [ 4600.895783] ? __pfx_kthread (kernel/kthread.c:341) [ 4600.895785] ret_from_fork (arch/x86/kernel/process.c:147) [ 4600.895787] ? __pfx_kthread (kernel/kthread.c:341) [ 4600.895788] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4600.895792] | [ 4672.449867] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4672.450228] rcu: 3-....: (4627297 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=1125574 | [ 4672.450791] rcu: (t=4629075 jiffies g=31081 q=2292 ncpus=4) [ 4672.451591] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4672.452214] Workqueue: netns cleanup_net [ 4672.452450] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4672.452761] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4672.453765] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 4672.454054] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 4672.454444] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 4672.454834] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 4672.455224] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 4672.455615] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 4672.456007] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 4672.456445] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4672.456764] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 4672.457154] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4672.457545] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4672.457933] PKRU: 55555554 [ 4672.458090] Call Trace: [ 4672.458235] [ 4672.458365] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4672.458614] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4672.458864] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 4672.459135] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 4672.459388] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4672.459637] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4672.459875] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4672.460167] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4672.460392] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4672.460657] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4672.460905] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4672.461143] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4672.461432] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4672.461709] [ 4672.461831] [ 4672.461954] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4672.462250] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4672.462522] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4672.462740] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4672.462948] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4672.463223] ? up_read (kernel/locking/rwsem.c:1622) [ 4672.463409] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4672.463656] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4672.463931] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4672.464195] __tcf_action_put (net/sched/act_api.c:377) [ 4672.464418] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4672.464643] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4672.464857] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4672.465147] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4672.465411] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4672.465666] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4672.465887] tcf_chain_flush (net/sched/cls_api.c:737) [ 4672.466102] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4672.466318] tcf_block_put (net/sched/cls_api.c:1543) [ 4672.466521] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4672.466768] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4672.466987] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4672.467191] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 4672.467496] default_device_exit_batch (net/core/dev.c:11629) [ 4672.467766] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4672.467971] ? process_one_work (kernel/workqueue.c:2606) [ 4672.468208] process_one_work (kernel/workqueue.c:2633) [ 4672.468438] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4672.468653] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4672.468894] kthread (kernel/kthread.c:388) [ 4672.469077] ? __pfx_kthread (kernel/kthread.c:341) [ 4672.469291] ret_from_fork (arch/x86/kernel/process.c:147) [ 4672.469501] ? __pfx_kthread (kernel/kthread.c:341) [ 4672.469716] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4699.198874] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 4655790 jiffies s: 6353 root: 0x8/. | [ 4699.199492] rcu: blocking rcu_node structures (internal RCU debug): | [ 4699.199825] Sending NMI from CPU 1 to CPUs 3: | [ 4699.200075] NMI backtrace for cpu 3 [ 4699.200082] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4699.200084] Workqueue: netns cleanup_net [ 4699.200091] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4699.200097] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4699.200099] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 4699.200100] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 4699.200101] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 4699.200102] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 4699.200103] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 4699.200104] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 4699.200106] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 4699.200107] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4699.200108] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 4699.200109] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4699.200109] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4699.200110] PKRU: 55555554 [ 4699.200111] Call Trace: [ 4699.200113] [ 4699.200115] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 4699.200120] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 4699.200124] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 4699.200127] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 4699.200131] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 4699.200133] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 4699.200137] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4699.200139] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4699.200141] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4699.200143] [ 4699.200144] [ 4699.200145] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4699.200149] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4699.200153] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4699.200158] ? up_read (kernel/locking/rwsem.c:1622) [ 4699.200161] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4699.200166] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4699.200167] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4699.200170] __tcf_action_put (net/sched/act_api.c:377) [ 4699.200175] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4699.200177] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4699.200179] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4699.200185] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4699.200191] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4699.200196] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4699.200199] tcf_chain_flush (net/sched/cls_api.c:737) [ 4699.200201] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4699.200204] tcf_block_put (net/sched/cls_api.c:1543) [ 4699.200207] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4699.200212] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4699.200216] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4699.200219] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 4699.200223] default_device_exit_batch (net/core/dev.c:11629) [ 4699.200226] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4699.200228] ? process_one_work (kernel/workqueue.c:2606) [ 4699.200232] process_one_work (kernel/workqueue.c:2633) [ 4699.200235] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4699.200238] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4699.200240] kthread (kernel/kthread.c:388) [ 4699.200242] ? __pfx_kthread (kernel/kthread.c:341) [ 4699.200243] ret_from_fork (arch/x86/kernel/process.c:147) [ 4699.200246] ? __pfx_kthread (kernel/kthread.c:341) [ 4699.200247] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4699.200253] | [ 4750.472866] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4750.473222] rcu: 3-....: (4705285 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=1144517 | [ 4750.473776] rcu: (t=4707098 jiffies g=31081 q=2292 ncpus=4) [ 4750.474568] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4750.475194] Workqueue: netns cleanup_net [ 4750.475422] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4750.475732] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4750.476740] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 4750.477031] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 4750.477424] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 4750.477816] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 4750.478212] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 4750.478603] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 4750.479003] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 4750.479449] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4750.479767] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 4750.480160] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4750.480554] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4750.480950] PKRU: 55555554 [ 4750.481104] Call Trace: [ 4750.481248] [ 4750.481370] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4750.481624] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4750.481880] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 4750.482160] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 4750.482415] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4750.482669] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4750.482916] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4750.483216] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4750.483446] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4750.483715] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4750.483970] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4750.484212] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4750.484508] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4750.484790] [ 4750.484914] [ 4750.485037] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4750.485338] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4750.485613] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4750.485836] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4750.486046] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4750.486326] ? up_read (kernel/locking/rwsem.c:1622) [ 4750.486514] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4750.486766] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4750.487043] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4750.487311] __tcf_action_put (net/sched/act_api.c:377) [ 4750.487533] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4750.487758] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4750.487974] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4750.488269] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4750.488537] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4750.488795] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4750.489039] tcf_chain_flush (net/sched/cls_api.c:737) [ 4750.489252] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4750.489471] tcf_block_put (net/sched/cls_api.c:1543) [ 4750.489676] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4750.489926] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4750.490146] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4750.490349] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 4750.490650] default_device_exit_batch (net/core/dev.c:11629) [ 4750.490918] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4750.491123] ? process_one_work (kernel/workqueue.c:2606) [ 4750.491357] process_one_work (kernel/workqueue.c:2633) [ 4750.491586] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4750.491798] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4750.492037] kthread (kernel/kthread.c:388) [ 4750.492220] ? __pfx_kthread (kernel/kthread.c:341) [ 4750.492433] ret_from_fork (arch/x86/kernel/process.c:147) [ 4750.492641] ? __pfx_kthread (kernel/kthread.c:341) [ 4750.492853] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4797.501877] rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: { 3-.... } 4754093 jiffies s: 6353 root: 0x8/. | [ 4797.502500] rcu: blocking rcu_node structures (internal RCU debug): | [ 4797.502834] Sending NMI from CPU 1 to CPUs 3: | [ 4797.503082] NMI backtrace for cpu 3 [ 4797.503088] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4797.503090] Workqueue: netns cleanup_net [ 4797.503098] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4797.503104] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4797.503106] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 4797.503107] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 4797.503108] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 4797.503109] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 4797.503110] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 4797.503111] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 4797.503114] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 4797.503115] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4797.503116] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 4797.503117] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4797.503118] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4797.503118] PKRU: 55555554 [ 4797.503119] Call Trace: [ 4797.503122] [ 4797.503124] ? nmi_cpu_backtrace (lib/nmi_backtrace.c:115) [ 4797.503129] ? nmi_cpu_backtrace_handler (arch/x86/kernel/apic/hw_nmi.c:48) [ 4797.503134] ? nmi_handle (arch/x86/kernel/nmi.c:150 (discriminator 7)) [ 4797.503137] ? default_do_nmi (arch/x86/kernel/nmi.c:351 (discriminator 22)) [ 4797.503141] ? exc_nmi (arch/x86/kernel/nmi.c:545) [ 4797.503143] ? end_repeat_nmi (arch/x86/entry/entry_64.S:1405) [ 4797.503147] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4797.503149] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4797.503152] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4797.503153] [ 4797.503154] [ 4797.503155] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4797.503159] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4797.503163] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4797.503168] ? up_read (kernel/locking/rwsem.c:1622) [ 4797.503171] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4797.503176] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4797.503178] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4797.503180] __tcf_action_put (net/sched/act_api.c:377) [ 4797.503185] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4797.503188] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4797.503190] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4797.503196] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4797.503202] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4797.503207] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4797.503210] tcf_chain_flush (net/sched/cls_api.c:737) [ 4797.503212] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4797.503215] tcf_block_put (net/sched/cls_api.c:1543) [ 4797.503218] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4797.503222] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4797.503226] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4797.503230] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 4797.503233] default_device_exit_batch (net/core/dev.c:11629) [ 4797.503236] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4797.503239] ? process_one_work (kernel/workqueue.c:2606) [ 4797.503242] process_one_work (kernel/workqueue.c:2633) [ 4797.503245] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4797.503247] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4797.503249] kthread (kernel/kthread.c:388) [ 4797.503252] ? __pfx_kthread (kernel/kthread.c:341) [ 4797.503253] ret_from_fork (arch/x86/kernel/process.c:147) [ 4797.503257] ? __pfx_kthread (kernel/kthread.c:341) [ 4797.503258] ret_from_fork_asm (arch/x86/entry/entry_64.S:250) | [ 4797.503264] | [ 4828.495866] rcu: INFO: rcu_preempt self-detected stall on CPU | [ 4828.496223] rcu: 3-....: (4783273 ticks this GP) idle=9224/1/0x4000000000000000 softirq=24881/24890 fqs=1163493 | [ 4828.496781] rcu: (t=4785121 jiffies g=31081 q=2293 ncpus=4) [ 4828.497573] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 4828.498190] Workqueue: netns cleanup_net [ 4828.498416] RIP: 0010:queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4828.498724] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 All code ======== 0: 54 push %rsp 1: 55 push %rbp 2: 48 89 fd mov %rdi,%rbp 5: 53 push %rbx 6: 66 90 xchg %ax,%ax 8: ba 01 00 00 00 mov $0x1,%edx d: 8b 45 00 mov 0x0(%rbp),%eax 10: 85 c0 test %eax,%eax 12: 75 14 jne 0x28 14: f0 0f b1 55 00 lock cmpxchg %edx,0x0(%rbp) 19: 85 c0 test %eax,%eax 1b: 75 f0 jne 0xd 1d: 5b pop %rbx 1e: 5d pop %rbp 1f: 41 5c pop %r12 21: 41 5d pop %r13 23: c3 ret 24: cc int3 25: cc int3 26: cc int3 27: cc int3 28: f3 90 pause 2a:* eb e1 jmp 0xd <-- trapping instruction 2c: 81 fe 00 01 00 00 cmp $0x100,%esi 32: 74 52 je 0x86 34: 40 30 f6 xor %sil,%sil 37: 85 f6 test %esi,%esi 39: 75 75 jne 0xb0 3b: f0 lock 3c: 0f .byte 0xf 3d: ba .byte 0xba 3e: 6d insl (%dx),%es:(%rdi) ... Code starting with the faulting instruction =========================================== 0: eb e1 jmp 0xffffffffffffffe3 2: 81 fe 00 01 00 00 cmp $0x100,%esi 8: 74 52 je 0x5c a: 40 30 f6 xor %sil,%sil d: 85 f6 test %esi,%esi f: 75 75 jne 0x86 11: f0 lock 12: 0f .byte 0xf 13: ba .byte 0xba 14: 6d insl (%dx),%es:(%rdi) ... [ 4828.499732] RSP: 0018:ffffb565000639d8 EFLAGS: 00000206 [ 4828.500023] RAX: 000000000581b690 RBX: ffff9b6b02518908 RCX: 0000000000000000 [ 4828.500417] RDX: 0000000000000001 RSI: 000000000581b690 RDI: ffff9b6b02518908 [ 4828.500811] RBP: ffff9b6b02518908 R08: 0000000000000001 R09: 0000000000000000 [ 4828.501202] R10: 0000000000000000 R11: ffffb56501677c98 R12: ffff9b6b02518900 [ 4828.501597] R13: 0000000000000000 R14: ffff9b6b02518908 R15: ffff9b6b02518968 [ 4828.501992] FS: 0000000000000000(0000) GS:ffff9b6b3ed80000(0000) knlGS:0000000000000000 [ 4828.502437] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4828.502758] CR2: 0000560e48897dec CR3: 0000000025460006 CR4: 0000000000770ef0 [ 4828.503152] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 4828.503548] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 4828.503947] PKRU: 55555554 [ 4828.504108] Call Trace: [ 4828.504254] [ 4828.504377] ? rcu_dump_cpu_stacks (./include/linux/cpumask.h:214 kernel/rcu/tree_stall.h:374) [ 4828.504630] ? rcu_sched_clock_irq (kernel/rcu/tree_stall.h:699 kernel/rcu/tree_stall.h:793 kernel/rcu/tree.c:3925 kernel/rcu/tree.c:2291) [ 4828.504888] ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194) [ 4828.505169] ? timekeeping_advance (kernel/time/timekeeping.c:2223 (discriminator 5)) [ 4828.505424] ? cpuacct_account_field (./include/linux/cgroup.h:434 kernel/sched/cpuacct.c:39 kernel/sched/cpuacct.c:354) [ 4828.505679] ? update_process_times (./arch/x86/include/asm/preempt.h:26 kernel/time/timer.c:2089) [ 4828.505926] ? __pfx_tick_nohz_highres_handler (kernel/time/tick-sched.c:1506) [ 4828.506226] ? tick_sched_handle (kernel/time/tick-sched.c:256) [ 4828.506458] ? tick_nohz_highres_handler (kernel/time/tick-sched.c:1528) [ 4828.506729] ? __hrtimer_run_queues (kernel/time/hrtimer.c:1688 kernel/time/hrtimer.c:1752) [ 4828.506982] ? hrtimer_interrupt (kernel/time/hrtimer.c:1817) [ 4828.507231] ? __sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1065 arch/x86/kernel/apic/apic.c:1082) [ 4828.507524] ? sysvec_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:1076 (discriminator 14)) [ 4828.507806] [ 4828.507931] [ 4828.508056] ? asm_sysvec_apic_timer_interrupt (./arch/x86/include/asm/idtentry.h:649) [ 4828.508352] ? queued_spin_lock_slowpath (./arch/x86/include/asm/vdso/processor.h:19 ./arch/x86/include/asm/qspinlock.h:99 kernel/locking/qspinlock.c:327) [ 4828.508630] do_raw_spin_lock (./include/asm-generic/qspinlock.h:114 kernel/locking/spinlock_debug.c:116) [ 4828.508852] __mutex_lock (kernel/locking/mutex.c:152 kernel/locking/mutex.c:626 kernel/locking/mutex.c:752) [ 4828.509064] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4828.509343] ? up_read (kernel/locking/rwsem.c:1622) [ 4828.509529] ? tc_setup_cb_destroy (net/sched/cls_api.c:3715) [ 4828.509781] ? refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4828.510057] refcount_dec_and_mutex_lock (./arch/x86/include/asm/atomic.h:103 ./include/linux/atomic/atomic-arch-fallback.h:949 ./include/linux/atomic/atomic-instrumented.h:401 ./include/linux/refcount.h:261 ./include/linux/refcount.h:304 ./include/linux/refcount.h:322 lib/refcount.c:119 lib/refcount.c:113) [ 4828.510325] __tcf_action_put (net/sched/act_api.c:377) [ 4828.510544] tcf_action_destroy (net/sched/act_api.c:413 net/sched/act_api.c:393 net/sched/act_api.c:1175) [ 4828.510771] tcf_exts_destroy (net/sched/cls_api.c:3312) [ 4828.510988] __fl_destroy_filter (./include/net/pkt_cls.h:261 net/sched/cls_flower.c:425) cls_flower [ 4828.511283] __fl_delete (net/sched/cls_flower.c:574) cls_flower [ 4828.511551] fl_destroy (net/sched/cls_flower.c:599) cls_flower [ 4828.511812] tcf_proto_destroy (net/sched/cls_api.c:418) [ 4828.512033] tcf_chain_flush (net/sched/cls_api.c:737) [ 4828.512250] __tcf_block_put (net/sched/cls_api.c:1070 net/sched/cls_api.c:1141 net/sched/cls_api.c:1305) [ 4828.512467] tcf_block_put (net/sched/cls_api.c:1543) [ 4828.512672] prio_destroy (net/sched/sch_prio.c:171) sch_prio [ 4828.512926] __qdisc_destroy (net/sched/sch_generic.c:1067) [ 4828.513148] dev_shutdown (net/sched/sch_generic.c:1483 (discriminator 9)) [ 4828.513353] unregister_netdevice_many_notify (./include/net/tcx.h:168 net/core/dev.c:11084) [ 4828.513658] default_device_exit_batch (net/core/dev.c:11629) [ 4828.513926] cleanup_net (net/core/net_namespace.c:636 (discriminator 3)) [ 4828.514132] ? process_one_work (kernel/workqueue.c:2606) [ 4828.514368] process_one_work (kernel/workqueue.c:2633) [ 4828.514593] worker_thread (kernel/workqueue.c:2700 kernel/workqueue.c:2787) [ 4828.514808] ? __pfx_worker_thread (kernel/workqueue.c:2733) [ 4828.515051] kthread (kernel/kthread.c:388) [ 4828.515233] ? __pfx_kthread (kernel/kthread.c:341) [ 4828.515445] ret_from_fork (arch/x86/kernel/process.c:147) [ 4828.515651] ? __pfx_kthread (kernel/kthread.c:341) Finger prints: __mutex_lock:refcount_dec_and_mutex_lock:__tcf_action_put:tcf_action_destroy queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock queued_spin_lock_slowpath:do_raw_spin_lock:__mutex_lock:refcount_dec_and_mutex_lock