[10236.420980] rcu: INFO: rcu_preempt self-detected stall on CPU [10236.421606] rcu: 0-....: (1 GPs behind) idle=7cf4/1/0x4000000000000000 softirq=41150/41152 fqs=2056816 [10236.422471] rcu: (t=9077704 jiffies g=77753 q=157 ncpus=4) [10236.422999] CPU: 0 PID: 5627 Comm: ncat Not tainted 6.8.0-rc2-virtme #1 [10236.423628] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [10236.424693] RIP: 0010:queued_spin_lock_slowpath+0x36/0x310 [10236.425211] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 <eb> e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 [10236.426921] RSP: 0000:ffffbef24018f688 EFLAGS: 00000202 [10236.427423] RAX: 0000000000000001 RBX: ffff9a9dc2ac0c98 RCX: 0000000000000001 [10236.428092] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9a9dc2ac0c98 [10236.428765] RBP: ffff9a9dc2ac0c98 R08: ffffbef24018f448 R09: 0000000000000020 [10236.429413] R10: ffffbef24018f538 R11: 0000000000000001 R12: 0000000000000000 [10236.460201] R13: ffff9a9dc2ac0c00 R14: ffff9a9dc2ac0c98 R15: ffff9a9dc1d5054c [10236.460869] FS: 00007fd2c23d9740(0000) GS:ffff9a9dfec00000(0000) knlGS:0000000000000000 [10236.461631] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [10236.462182] CR2: 0000559104c93000 CR3: 0000000003d4e005 CR4: 0000000000770ef0 [10236.462842] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [10236.463492] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [10236.464167] PKRU: 55555554 [10236.464423] Call Trace: [10236.464660] <IRQ> [10236.464868] ? rcu_dump_cpu_stacks+0xfa/0x180 [10236.465297] ? rcu_sched_clock_irq+0x7bb/0x13d0 [10236.465737] ? lock_is_held_type+0x9f/0x120 [10236.466144] ? lock_is_held_type+0x9f/0x120 [10236.466555] ? update_process_times+0x5f/0x90 [10236.466983] ? __pfx_tick_nohz_highres_handler+0x10/0x10 [10236.467492] ? tick_sched_handle+0x33/0x50 [10236.467890] ? tick_nohz_highres_handler+0x71/0x90 [10236.468352] ? __hrtimer_run_queues+0x86/0x390 [10236.468799] ? hrtimer_interrupt+0x100/0x240 [10236.469218] ? __sysvec_apic_timer_interrupt+0x66/0x1a0 [10236.469726] ? sysvec_apic_timer_interrupt+0x73/0x90 [10236.510314] </IRQ> [10236.510527] <TASK> [10236.510740] ? asm_sysvec_apic_timer_interrupt+0x1a/0x20 [10236.511258] ? queued_spin_lock_slowpath+0x36/0x310 [10236.511715] do_raw_spin_lock+0x9b/0xa0 [10236.512077] tcp_v4_rcv+0xdb5/0xef0 [10236.512399] ip_protocol_deliver_rcu+0x4b/0x1e0 [10236.512801] ip_local_deliver_finish+0xaf/0x180 [10236.513199] __netif_receive_skb_one_core+0x8b/0xa0 [10236.513640] netif_receive_skb+0x87/0x2c0 [10236.514006] tcf_mirred_to_dev+0x1f6/0x410 [act_mirred] [10236.514469] tcf_mirred_act+0xee/0x560 [act_mirred] [10236.514906] ? tcf_skbedit_act+0x1d6/0x410 [act_skbedit] [10236.515384] ? lock_acquire+0xc1/0x2b0 [10236.515729] tcf_action_exec+0xac/0x230 [10236.516079] fl_classify+0x20e/0x260 [cls_flower] [10236.516514] ? __bfs+0xf3/0x240 [10236.516814] ? check_irq_usage+0x19c/0xc40 [10236.517207] ? __lock_acquire+0xa6e/0x1750 [10236.517591] ? lock_acquire+0x179/0x2b0 [10236.517944] ? check_path.constprop.0+0x24/0x50 [10236.518363] ? check_noncircular+0x81/0x150 [10236.518774] ? lock_acquire+0x179/0x2b0 [10236.519177] ? is_bpf_text_address+0x71/0x100 [10236.519563] ? lock_release+0x1d3/0x270 [10236.519889] ? __lock_acquire+0xa6e/0x1750 [10236.540393] tcf_classify+0x17a/0x540 [10236.540755] tc_run+0x9f/0x150 [10236.541067] __dev_queue_xmit+0x311/0x1180 [10236.541448] ? mark_held_locks+0x49/0x80 [10236.541819] ? eth_header+0x2a/0xc0 [10236.542170] ip_finish_output2+0x22d/0xa20 [10236.542572] ? __ip_queue_xmit+0x1d5/0x6f0 [10236.542971] __ip_queue_xmit+0x1d5/0x6f0 [10236.543367] __tcp_transmit_skb+0xb65/0xd30 [10236.543793] ? __alloc_skb+0xdc/0x1a0 [10236.544180] ? __pfx_tcp_delack_timer+0x10/0x10 [10236.544651] tcp_delack_timer_handler+0xa3/0xf0 [10236.545124] tcp_delack_timer+0x5a/0x140 [10236.545540] ? __pfx_tcp_delack_timer+0x10/0x10 [10236.546000] call_timer_fn+0xa5/0x220 [10236.546388] __run_timers.part.0+0x1f8/0x2c0 [10236.546831] ? kvm_clock_get_cycles+0x18/0x30 [10236.547257] run_timer_softirq+0x2a/0x50 [10236.547635] __do_softirq+0xc8/0x37a [10236.547993] irq_exit_rcu+0x7f/0xa0 [10236.548334] sysvec_apic_timer_interrupt+0x41/0x90 [10236.548798] asm_sysvec_apic_timer_interrupt+0x1a/0x20 [10236.549291] RIP: 0033:0x7fd2c2613700 [10236.549654] Code: ff ff 64 89 02 eb bd 0f 1f 00 f3 0f 1e fa 41 89 ca 64 8b 04 25 18 00 00 00 85 c0 75 1d 45 31 c9 45 31 c0 b8 2c 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 68 c3 0f 1f 80 00 00 00 00 41 54 48 83 ec 20 [10236.571476] RSP: 002b:00007ffe6ba49238 EFLAGS: 00000246 [10236.571971] RAX: 0000000000002000 RBX: 0000000000002000 RCX: 00007fd2c2613700 [10236.572647] RDX: 0000000000002000 RSI: 0000559104c92780 RDI: 0000000000000005 [10236.573293] RBP: 0000559104b589e0 R08: 0000000000000000 R09: 0000000000000000 [10236.573943] R10: 0000000000000000 R11: 0000000000000246 R12: 0000559104bbccf0 [10236.574580] R13: 0000559104bbcb10 R14: 0000559104bbcb10 R15: 0000559104b589e0 [10236.575239] </TASK> [10236.575475] Sending NMI from CPU 0 to CPUs 2: [10236.575911] NMI backtrace for cpu 2 [10236.575917] CPU: 2 PID: 5626 Comm: ncat Not tainted 6.8.0-rc2-virtme #1 [10236.575920] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [10236.575922] RIP: 0010:queued_spin_lock_slowpath+0x36/0x310 [10236.575932] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 <eb> e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 [10236.575934] RSP: 0018:ffffbef2401077a8 EFLAGS: 00000202 [10236.575937] RAX: 0000000000000001 RBX: ffff9a9dc2ac0098 RCX: 0000000000000001 [10236.575938] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9a9dc2ac0098 [10236.575940] RBP: ffff9a9dc2ac0098 R08: 0000000000000008 R09: ffffffff96d9d1a8 [10236.575941] R10: 0000000000000000 R11: 0000000000000001 R12: 0000000000000000 [10236.575942] R13: ffff9a9dc2ac0000 R14: ffff9a9dc2ac0098 R15: ffff9a9dc23ea94c [10236.575947] FS: 00007fa251394740(0000) GS:ffff9a9dfed00000(0000) knlGS:0000000000000000 [10236.575949] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [10236.575950] CR2: 00005653cd1c7340 CR3: 000000000263c001 CR4: 0000000000770ef0 [10236.575952] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [10236.575953] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [10236.575955] PKRU: 55555554 [10236.575956] Call Trace: [10236.575959] <NMI> [10236.575963] ? nmi_cpu_backtrace+0xa2/0x110 [10236.575970] ? nmi_cpu_backtrace_handler+0x11/0x20 [10236.575975] ? nmi_handle+0xd0/0x270 [10236.575979] ? default_do_nmi+0x41/0x100 [10236.575984] ? exc_nmi+0xf3/0x120 [10236.575988] ? end_repeat_nmi+0xf/0x60 [10236.575997] ? queued_spin_lock_slowpath+0x36/0x310 [10236.576001] ? queued_spin_lock_slowpath+0x36/0x310 [10236.576005] ? queued_spin_lock_slowpath+0x36/0x310 [10236.576008] </NMI> [10236.576009] <TASK> [10236.576011] do_raw_spin_lock+0x9b/0xa0 [10236.576017] tcp_v4_rcv+0xdb5/0xef0 [10236.576025] ip_protocol_deliver_rcu+0x4b/0x1e0 [10236.576030] ip_local_deliver_finish+0xaf/0x180 [10236.576033] __netif_receive_skb_one_core+0x8b/0xa0 [10236.576039] netif_receive_skb+0x87/0x2c0 [10236.576044] tcf_mirred_to_dev+0x1f6/0x410 [act_mirred] [10236.576052] tcf_mirred_act+0xee/0x560 [act_mirred] [10236.576059] ? tcf_skbedit_act+0x1d6/0x410 [act_skbedit] [10236.576066] tcf_action_exec+0xac/0x230 [10236.576073] fl_classify+0x20e/0x260 [cls_flower] [10236.576084] ? kernel_text_address+0xce/0xe0 [10236.576089] ? __kernel_text_address+0x12/0x40 [10236.576091] ? unwind_get_return_address+0x23/0x40 [10236.576095] ? __pfx_stack_trace_consume_entry+0x10/0x10 [10236.576100] ? arch_stack_walk+0xa2/0x100 [10236.576108] ? stack_trace_save+0x4f/0x80 [10236.576112] ? save_trace+0x3f/0x3b0 [10236.576117] ? __lock_acquire+0xa6e/0x1750 [10236.576123] tcf_classify+0x17a/0x540 [10236.576129] tc_run+0x9f/0x150 [10236.576135] __dev_queue_xmit+0x311/0x1180 [10236.576139] ? mark_held_locks+0x49/0x80 [10236.576141] ? eth_header+0x2a/0xc0 [10236.576147] ip_finish_output2+0x22d/0xa20 [10236.576150] ? ip_skb_dst_mtu+0x51/0x1c0 [10236.576155] ? __ip_queue_xmit+0x1d5/0x6f0 [10236.576157] __ip_queue_xmit+0x1d5/0x6f0 [10236.576162] __tcp_transmit_skb+0xb65/0xd30 [10236.576167] ? __alloc_skb+0xdc/0x1a0 [10236.576172] release_sock+0x4f/0xb0 [10236.576176] inet_shutdown+0xa8/0x100 [10236.576181] __sys_shutdown+0x45/0x90 [10236.576186] __x64_sys_shutdown+0x14/0x20 [10236.576188] do_syscall_64+0xc9/0x1e0 [10236.576193] entry_SYSCALL_64_after_hwframe+0x6f/0x77 [10236.576195] RIP: 0033:0x7fa2514bebeb [10236.576211] Code: 73 01 c3 48 8b 0d 15 92 1b 00 f7 d8 64 89 01 48 83 c8 ff c3 66 2e 0f 1f 84 00 00 00 00 00 90 f3 0f 1e fa b8 30 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d e5 91 1b 00 f7 d8 64 89 01 48 [10236.576214] RSP: 002b:00007ffe4efd4718 EFLAGS: 00000246 ORIG_RAX: 0000000000000030 [10236.576216] RAX: ffffffffffffffda RBX: 0000000000000008 RCX: 00007fa2514bebeb [10236.576218] RDX: 00005653cd662420 RSI: 0000000000000001 RDI: 0000000000000008 [10236.576219] RBP: 0000000000000008 R08: 0000000000000000 R09: 0000000000000000 [10236.576221] R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000008 [10236.576222] R13: 0000000000000000 R14: 0000000000000001 R15: 0000000000000000 [10236.576228] </TASK> [10314.603726] rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: [10314.604292] rcu: 0-....: (1 GPs behind) idle=7cf4/1/0x4000000000000000 softirq=41150/41152 fqs=2072458 [10314.605053] rcu: 2-....: (1 GPs behind) idle=926c/1/0x4000000000000000 softirq=41281/41282 fqs=2072458 [10314.605815] rcu: (detected by 3, t=9155887 jiffies, g=77753, q=157 ncpus=4) [10314.606403] Sending NMI from CPU 3 to CPUs 0: [10314.608813] NMI backtrace for cpu 0 [10314.608824] CPU: 0 PID: 5627 Comm: ncat Not tainted 6.8.0-rc2-virtme #1 [10314.608828] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [10314.608834] RIP: 0010:queued_spin_lock_slowpath+0x19/0x310 [10314.608848] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 <8b> 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d [10314.608851] RSP: 0000:ffffbef24018f688 EFLAGS: 00000202 [10314.608854] RAX: 0000000000000001 RBX: ffff9a9dc2ac0c98 RCX: 0000000000000001 [10314.608856] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9a9dc2ac0c98 [10314.608857] RBP: ffff9a9dc2ac0c98 R08: ffffbef24018f448 R09: 0000000000000020 [10314.608859] R10: ffffbef24018f538 R11: 0000000000000001 R12: 0000000000000000 [10314.608861] R13: ffff9a9dc2ac0c00 R14: ffff9a9dc2ac0c98 R15: ffff9a9dc1d5054c [10314.608866] FS: 00007fd2c23d9740(0000) GS:ffff9a9dfec00000(0000) knlGS:0000000000000000 [10314.608868] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [10314.608870] CR2: 0000559104c93000 CR3: 0000000003d4e005 CR4: 0000000000770ef0 [10314.608871] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [10314.608873] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [10314.608875] PKRU: 55555554 [10314.608876] Call Trace: [10314.608886] <NMI> [10314.608896] ? nmi_cpu_backtrace+0xa2/0x110 [10314.608904] ? nmi_cpu_backtrace_handler+0x11/0x20 [10314.608911] ? nmi_handle+0xd0/0x270 [10314.608916] ? default_do_nmi+0x41/0x100 [10314.608922] ? exc_nmi+0xf3/0x120 [10314.608925] ? end_repeat_nmi+0xf/0x60 [10314.608935] ? queued_spin_lock_slowpath+0x19/0x310 [10314.608939] ? queued_spin_lock_slowpath+0x19/0x310 [10314.608942] ? queued_spin_lock_slowpath+0x19/0x310 [10314.608946] </NMI> [10314.608947] <TASK> [10314.608948] do_raw_spin_lock+0x9b/0xa0 [10314.608956] tcp_v4_rcv+0xdb5/0xef0 [10314.608966] ip_protocol_deliver_rcu+0x4b/0x1e0 [10314.608973] ip_local_deliver_finish+0xaf/0x180 [10314.608976] __netif_receive_skb_one_core+0x8b/0xa0 [10314.608983] netif_receive_skb+0x87/0x2c0 [10314.608988] tcf_mirred_to_dev+0x1f6/0x410 [act_mirred] [10314.608998] tcf_mirred_act+0xee/0x560 [act_mirred] [10314.609005] ? tcf_skbedit_act+0x1d6/0x410 [act_skbedit] [10314.609011] ? lock_acquire+0xc1/0x2b0 [10314.609016] tcf_action_exec+0xac/0x230 [10314.609023] fl_classify+0x20e/0x260 [cls_flower] [10314.609035] ? __bfs+0xf3/0x240 [10314.609039] ? check_irq_usage+0x19c/0xc40 [10314.609042] ? __lock_acquire+0xa6e/0x1750 [10314.609045] ? lock_acquire+0x179/0x2b0 [10314.609049] ? check_path.constprop.0+0x24/0x50 [10314.609052] ? check_noncircular+0x81/0x150 [10314.609054] ? lock_acquire+0x179/0x2b0 [10314.609058] ? is_bpf_text_address+0x71/0x100 [10314.609062] ? lock_release+0x1d3/0x270 [10314.609066] ? __lock_acquire+0xa6e/0x1750 [10314.609073] tcf_classify+0x17a/0x540 [10314.609079] tc_run+0x9f/0x150 [10314.609085] __dev_queue_xmit+0x311/0x1180 [10314.609089] ? mark_held_locks+0x49/0x80 [10314.609092] ? eth_header+0x2a/0xc0 [10314.609098] ip_finish_output2+0x22d/0xa20 [10314.609105] ? __ip_queue_xmit+0x1d5/0x6f0 [10314.609108] __ip_queue_xmit+0x1d5/0x6f0 [10314.609113] __tcp_transmit_skb+0xb65/0xd30 [10314.609119] ? __alloc_skb+0xdc/0x1a0 [10314.609124] ? __pfx_tcp_delack_timer+0x10/0x10 [10314.609126] tcp_delack_timer_handler+0xa3/0xf0 [10314.609130] tcp_delack_timer+0x5a/0x140 [10314.609132] ? __pfx_tcp_delack_timer+0x10/0x10 [10314.609135] call_timer_fn+0xa5/0x220 [10314.609140] __run_timers.part.0+0x1f8/0x2c0 [10314.609145] ? kvm_clock_get_cycles+0x18/0x30 [10314.609152] run_timer_softirq+0x2a/0x50 [10314.609154] __do_softirq+0xc8/0x37a [10314.609160] irq_exit_rcu+0x7f/0xa0 [10314.609165] sysvec_apic_timer_interrupt+0x41/0x90 [10314.609169] asm_sysvec_apic_timer_interrupt+0x1a/0x20 [10314.609172] RIP: 0033:0x7fd2c2613700 [10314.609176] Code: ff ff 64 89 02 eb bd 0f 1f 00 f3 0f 1e fa 41 89 ca 64 8b 04 25 18 00 00 00 85 c0 75 1d 45 31 c9 45 31 c0 b8 2c 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 68 c3 0f 1f 80 00 00 00 00 41 54 48 83 ec 20 [10314.609179] RSP: 002b:00007ffe6ba49238 EFLAGS: 00000246 [10314.609182] RAX: 0000000000002000 RBX: 0000000000002000 RCX: 00007fd2c2613700 [10314.609184] RDX: 0000000000002000 RSI: 0000559104c92780 RDI: 0000000000000005 [10314.609186] RBP: 0000559104b589e0 R08: 0000000000000000 R09: 0000000000000000 [10314.609187] R10: 0000000000000000 R11: 0000000000000246 R12: 0000559104bbccf0 [10314.609189] R13: 0000559104bbcb10 R14: 0000559104bbcb10 R15: 0000559104b589e0 [10314.609196] </TASK> [10314.609779] Sending NMI from CPU 3 to CPUs 2: [10314.684488] NMI backtrace for cpu 2 [10314.684493] CPU: 2 PID: 5626 Comm: ncat Not tainted 6.8.0-rc2-virtme #1 [10314.684497] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [10314.684499] RIP: 0010:queued_spin_lock_slowpath+0x36/0x310 [10314.684508] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 <eb> e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 [10314.684511] RSP: 0018:ffffbef2401077a8 EFLAGS: 00000202 [10314.684513] RAX: 0000000000000001 RBX: ffff9a9dc2ac0098 RCX: 0000000000000001 [10314.684515] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9a9dc2ac0098 [10314.684517] RBP: ffff9a9dc2ac0098 R08: 0000000000000008 R09: ffffffff96d9d1a8 [10314.684518] R10: 0000000000000000 R11: 0000000000000001 R12: 0000000000000000 [10314.684520] R13: ffff9a9dc2ac0000 R14: ffff9a9dc2ac0098 R15: ffff9a9dc23ea94c [10314.684524] FS: 00007fa251394740(0000) GS:ffff9a9dfed00000(0000) knlGS:0000000000000000 [10314.684526] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [10314.684528] CR2: 00005653cd1c7340 CR3: 000000000263c001 CR4: 0000000000770ef0 [10314.684529] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [10314.684530] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [10314.684532] PKRU: 55555554 [10314.684533] Call Trace: [10314.684535] <NMI> [10314.684540] ? nmi_cpu_backtrace+0xa2/0x110 [10314.684548] ? nmi_cpu_backtrace_handler+0x11/0x20 [10314.684554] ? nmi_handle+0xd0/0x270 [10314.684560] ? default_do_nmi+0x41/0x100 [10314.684565] ? exc_nmi+0xf3/0x120 [10314.684568] ? end_repeat_nmi+0xf/0x60 [10314.684577] ? queued_spin_lock_slowpath+0x36/0x310 [10314.684581] ? queued_spin_lock_slowpath+0x36/0x310 [10314.684585] ? queued_spin_lock_slowpath+0x36/0x310 [10314.684589] </NMI> [10314.684590] <TASK> [10314.684591] do_raw_spin_lock+0x9b/0xa0 [10314.684597] tcp_v4_rcv+0xdb5/0xef0 [10314.684605] ip_protocol_deliver_rcu+0x4b/0x1e0 [10314.684611] ip_local_deliver_finish+0xaf/0x180 [10314.684614] __netif_receive_skb_one_core+0x8b/0xa0 [10314.684620] netif_receive_skb+0x87/0x2c0 [10314.684625] tcf_mirred_to_dev+0x1f6/0x410 [act_mirred] [10314.684634] tcf_mirred_act+0xee/0x560 [act_mirred] [10314.684640] ? tcf_skbedit_act+0x1d6/0x410 [act_skbedit] [10314.684648] tcf_action_exec+0xac/0x230 [10314.684655] fl_classify+0x20e/0x260 [cls_flower] [10314.684666] ? kernel_text_address+0xce/0xe0 [10314.684670] ? __kernel_text_address+0x12/0x40 [10314.684673] ? unwind_get_return_address+0x23/0x40 [10314.684676] ? __pfx_stack_trace_consume_entry+0x10/0x10 [10314.684680] ? arch_stack_walk+0xa2/0x100 [10314.684690] ? stack_trace_save+0x4f/0x80 [10314.684694] ? save_trace+0x3f/0x3b0 [10314.684698] ? __lock_acquire+0xa6e/0x1750 [10314.684705] tcf_classify+0x17a/0x540 [10314.684710] tc_run+0x9f/0x150 [10314.684717] __dev_queue_xmit+0x311/0x1180 [10314.684720] ? mark_held_locks+0x49/0x80 [10314.684723] ? eth_header+0x2a/0xc0 [10314.684729] ip_finish_output2+0x22d/0xa20 [10314.684733] ? ip_skb_dst_mtu+0x51/0x1c0 [10314.684738] ? __ip_queue_xmit+0x1d5/0x6f0 [10314.684740] __ip_queue_xmit+0x1d5/0x6f0 [10314.684745] __tcp_transmit_skb+0xb65/0xd30 [10314.684752] ? __alloc_skb+0xdc/0x1a0 [10314.684758] release_sock+0x4f/0xb0 [10314.684761] inet_shutdown+0xa8/0x100 [10314.684765] __sys_shutdown+0x45/0x90 [10314.684769] __x64_sys_shutdown+0x14/0x20 [10314.684771] do_syscall_64+0xc9/0x1e0 [10314.684776] entry_SYSCALL_64_after_hwframe+0x6f/0x77 [10314.684779] RIP: 0033:0x7fa2514bebeb [10314.684782] Code: 73 01 c3 48 8b 0d 15 92 1b 00 f7 d8 64 89 01 48 83 c8 ff c3 66 2e 0f 1f 84 00 00 00 00 00 90 f3 0f 1e fa b8 30 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d e5 91 1b 00 f7 d8 64 89 01 48 [10314.684785] RSP: 002b:00007ffe4efd4718 EFLAGS: 00000246 ORIG_RAX: 0000000000000030 [10314.684787] RAX: ffffffffffffffda RBX: 0000000000000008 RCX: 00007fa2514bebeb [10314.684788] RDX: 00005653cd662420 RSI: 0000000000000001 RDI: 0000000000000008 [10314.684790] RBP: 0000000000000008 R08: 0000000000000000 R09: 0000000000000000 [10314.684791] R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000008 [10314.684793] R13: 0000000000000000 R14: 0000000000000001 R15: 0000000000000000 [10314.684799] </TASK> [10392.688712] rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: [10392.689347] rcu: 0-....: (1 GPs behind) idle=7cf4/1/0x4000000000000000 softirq=41150/41152 fqs=2088015 [10392.690176] rcu: 2-....: (1 GPs behind) idle=926c/1/0x4000000000000000 softirq=41281/41282 fqs=2088015 [10392.690973] rcu: (detected by 1, t=9233974 jiffies, g=77753, q=157 ncpus=4) [10392.691567] Sending NMI from CPU 1 to CPUs 0: [10392.691975] NMI backtrace for cpu 0 [10392.691983] CPU: 0 PID: 5627 Comm: ncat Not tainted 6.8.0-rc2-virtme #1 [10392.691987] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [10392.691989] RIP: 0010:queued_spin_lock_slowpath+0x36/0x310 [10392.692012] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 <eb> e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 [10392.692015] RSP: 0000:ffffbef24018f688 EFLAGS: 00000202 [10392.692017] RAX: 0000000000000001 RBX: ffff9a9dc2ac0c98 RCX: 0000000000000001 [10392.692019] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9a9dc2ac0c98 [10392.692021] RBP: ffff9a9dc2ac0c98 R08: ffffbef24018f448 R09: 0000000000000020 [10392.692022] R10: ffffbef24018f538 R11: 0000000000000001 R12: 0000000000000000 [10392.692023] R13: ffff9a9dc2ac0c00 R14: ffff9a9dc2ac0c98 R15: ffff9a9dc1d5054c [10392.692027] FS: 00007fd2c23d9740(0000) GS:ffff9a9dfec00000(0000) knlGS:0000000000000000 [10392.692029] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [10392.692030] CR2: 0000559104c93000 CR3: 0000000003d4e005 CR4: 0000000000770ef0 [10392.692032] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [10392.692033] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [10392.692035] PKRU: 55555554 [10392.692036] Call Trace: [10392.692039] <NMI> [10392.692045] ? nmi_cpu_backtrace+0xa2/0x110 [10392.692053] ? nmi_cpu_backtrace_handler+0x11/0x20 [10392.692061] ? nmi_handle+0xd0/0x270 [10392.692065] ? default_do_nmi+0x41/0x100 [10392.692072] ? exc_nmi+0xf3/0x120 [10392.692075] ? end_repeat_nmi+0xf/0x60 [10392.692084] ? queued_spin_lock_slowpath+0x36/0x310 [10392.692088] ? queued_spin_lock_slowpath+0x36/0x310 [10392.692091] ? queued_spin_lock_slowpath+0x36/0x310 [10392.692095] </NMI> [10392.692095] <TASK> [10392.692097] do_raw_spin_lock+0x9b/0xa0 [10392.692104] tcp_v4_rcv+0xdb5/0xef0 [10392.692114] ip_protocol_deliver_rcu+0x4b/0x1e0 [10392.692119] ip_local_deliver_finish+0xaf/0x180 [10392.692122] __netif_receive_skb_one_core+0x8b/0xa0 [10392.692129] netif_receive_skb+0x87/0x2c0 [10392.692134] tcf_mirred_to_dev+0x1f6/0x410 [act_mirred] [10392.692147] tcf_mirred_act+0xee/0x560 [act_mirred] [10392.692153] ? tcf_skbedit_act+0x1d6/0x410 [act_skbedit] [10392.692158] ? lock_acquire+0xc1/0x2b0 [10392.692162] tcf_action_exec+0xac/0x230 [10392.692169] fl_classify+0x20e/0x260 [cls_flower] [10392.692180] ? __bfs+0xf3/0x240 [10392.692184] ? check_irq_usage+0x19c/0xc40 [10392.692186] ? __lock_acquire+0xa6e/0x1750 [10392.692189] ? lock_acquire+0x179/0x2b0 [10392.692193] ? check_path.constprop.0+0x24/0x50 [10392.692196] ? check_noncircular+0x81/0x150 [10392.692198] ? lock_acquire+0x179/0x2b0 [10392.692201] ? is_bpf_text_address+0x71/0x100 [10392.692206] ? lock_release+0x1d3/0x270 [10392.692210] ? __lock_acquire+0xa6e/0x1750 [10392.692215] tcf_classify+0x17a/0x540 [10392.692222] tc_run+0x9f/0x150 [10392.692229] __dev_queue_xmit+0x311/0x1180 [10392.692233] ? mark_held_locks+0x49/0x80 [10392.692236] ? eth_header+0x2a/0xc0 [10392.692241] ip_finish_output2+0x22d/0xa20 [10392.692248] ? __ip_queue_xmit+0x1d5/0x6f0 [10392.692251] __ip_queue_xmit+0x1d5/0x6f0 [10392.692256] __tcp_transmit_skb+0xb65/0xd30 [10392.692263] ? __alloc_skb+0xdc/0x1a0 [10392.692268] ? __pfx_tcp_delack_timer+0x10/0x10 [10392.692270] tcp_delack_timer_handler+0xa3/0xf0 [10392.692273] tcp_delack_timer+0x5a/0x140 [10392.692276] ? __pfx_tcp_delack_timer+0x10/0x10 [10392.692278] call_timer_fn+0xa5/0x220 [10392.692283] __run_timers.part.0+0x1f8/0x2c0 [10392.692287] ? kvm_clock_get_cycles+0x18/0x30 [10392.692295] run_timer_softirq+0x2a/0x50 [10392.692297] __do_softirq+0xc8/0x37a [10392.692302] irq_exit_rcu+0x7f/0xa0 [10392.692306] sysvec_apic_timer_interrupt+0x41/0x90 [10392.692310] asm_sysvec_apic_timer_interrupt+0x1a/0x20 [10392.692313] RIP: 0033:0x7fd2c2613700 [10392.692322] Code: ff ff 64 89 02 eb bd 0f 1f 00 f3 0f 1e fa 41 89 ca 64 8b 04 25 18 00 00 00 85 c0 75 1d 45 31 c9 45 31 c0 b8 2c 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 68 c3 0f 1f 80 00 00 00 00 41 54 48 83 ec 20 [10392.692324] RSP: 002b:00007ffe6ba49238 EFLAGS: 00000246 [10392.692326] RAX: 0000000000002000 RBX: 0000000000002000 RCX: 00007fd2c2613700 [10392.692328] RDX: 0000000000002000 RSI: 0000559104c92780 RDI: 0000000000000005 [10392.692329] RBP: 0000559104b589e0 R08: 0000000000000000 R09: 0000000000000000 [10392.692330] R10: 0000000000000000 R11: 0000000000000246 R12: 0000559104bbccf0 [10392.692332] R13: 0000559104bbcb10 R14: 0000559104bbcb10 R15: 0000559104b589e0 [10392.692337] </TASK> [10392.692964] Sending NMI from CPU 1 to CPUs 2: [10392.849376] NMI backtrace for cpu 2 [10392.849383] CPU: 2 PID: 5626 Comm: ncat Not tainted 6.8.0-rc2-virtme #1 [10392.849387] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [10392.849389] RIP: 0010:queued_spin_lock_slowpath+0x34/0x310 [10392.849399] Code: 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc <f3> 90 eb e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba [10392.849401] RSP: 0018:ffffbef2401077a8 EFLAGS: 00000202 [10392.849404] RAX: 0000000000000001 RBX: ffff9a9dc2ac0098 RCX: 0000000000000001 [10392.849405] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9a9dc2ac0098 [10392.849407] RBP: ffff9a9dc2ac0098 R08: 0000000000000008 R09: ffffffff96d9d1a8 [10392.849408] R10: 0000000000000000 R11: 0000000000000001 R12: 0000000000000000 [10392.849409] R13: ffff9a9dc2ac0000 R14: ffff9a9dc2ac0098 R15: ffff9a9dc23ea94c [10392.849414] FS: 00007fa251394740(0000) GS:ffff9a9dfed00000(0000) knlGS:0000000000000000 [10392.849417] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [10392.849418] CR2: 00005653cd1c7340 CR3: 000000000263c001 CR4: 0000000000770ef0 [10392.849420] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [10392.849421] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [10392.849423] PKRU: 55555554 [10392.849424] Call Trace: [10392.849427] <NMI> [10392.849433] ? nmi_cpu_backtrace+0xa2/0x110 [10392.849442] ? nmi_cpu_backtrace_handler+0x11/0x20 [10392.849448] ? nmi_handle+0xd0/0x270 [10392.849453] ? default_do_nmi+0x41/0x100 [10392.849459] ? exc_nmi+0xf3/0x120 [10392.849462] ? end_repeat_nmi+0xf/0x60 [10392.849470] ? queued_spin_lock_slowpath+0x34/0x310 [10392.849474] ? queued_spin_lock_slowpath+0x34/0x310 [10392.849478] ? queued_spin_lock_slowpath+0x34/0x310 [10392.849481] </NMI> [10392.849482] <TASK> [10392.849484] do_raw_spin_lock+0x9b/0xa0 [10392.849490] tcp_v4_rcv+0xdb5/0xef0 [10392.849498] ip_protocol_deliver_rcu+0x4b/0x1e0 [10392.849504] ip_local_deliver_finish+0xaf/0x180 [10392.849506] __netif_receive_skb_one_core+0x8b/0xa0 [10392.849513] netif_receive_skb+0x87/0x2c0 [10392.849518] tcf_mirred_to_dev+0x1f6/0x410 [act_mirred] [10392.849527] tcf_mirred_act+0xee/0x560 [act_mirred] [10392.849533] ? tcf_skbedit_act+0x1d6/0x410 [act_skbedit] [10392.849540] tcf_action_exec+0xac/0x230 [10392.849546] fl_classify+0x20e/0x260 [cls_flower] [10392.849557] ? kernel_text_address+0xce/0xe0 [10392.849561] ? __kernel_text_address+0x12/0x40 [10392.849564] ? unwind_get_return_address+0x23/0x40 [10392.849567] ? __pfx_stack_trace_consume_entry+0x10/0x10 [10392.849573] ? arch_stack_walk+0xa2/0x100 [10392.849579] ? stack_trace_save+0x4f/0x80 [10392.849583] ? save_trace+0x3f/0x3b0 [10392.849599] ? __lock_acquire+0xa6e/0x1750 [10392.849606] tcf_classify+0x17a/0x540 [10392.849612] tc_run+0x9f/0x150 [10392.849618] __dev_queue_xmit+0x311/0x1180 [10392.849622] ? mark_held_locks+0x49/0x80 [10392.849624] ? eth_header+0x2a/0xc0 [10392.849629] ip_finish_output2+0x22d/0xa20 [10392.849634] ? ip_skb_dst_mtu+0x51/0x1c0 [10392.849638] ? __ip_queue_xmit+0x1d5/0x6f0 [10392.849641] __ip_queue_xmit+0x1d5/0x6f0 [10392.849646] __tcp_transmit_skb+0xb65/0xd30 [10392.849652] ? __alloc_skb+0xdc/0x1a0 [10392.849657] release_sock+0x4f/0xb0 [10392.849660] inet_shutdown+0xa8/0x100 [10392.849665] __sys_shutdown+0x45/0x90 [10392.849670] __x64_sys_shutdown+0x14/0x20 [10392.849672] do_syscall_64+0xc9/0x1e0 [10392.849676] entry_SYSCALL_64_after_hwframe+0x6f/0x77 [10392.849679] RIP: 0033:0x7fa2514bebeb [10392.849682] Code: 73 01 c3 48 8b 0d 15 92 1b 00 f7 d8 64 89 01 48 83 c8 ff c3 66 2e 0f 1f 84 00 00 00 00 00 90 f3 0f 1e fa b8 30 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d e5 91 1b 00 f7 d8 64 89 01 48 [10392.849685] RSP: 002b:00007ffe4efd4718 EFLAGS: 00000246 ORIG_RAX: 0000000000000030 [10392.849687] RAX: ffffffffffffffda RBX: 0000000000000008 RCX: 00007fa2514bebeb [10392.849689] RDX: 00005653cd662420 RSI: 0000000000000001 RDI: 0000000000000008 [10392.849690] RBP: 0000000000000008 R08: 0000000000000000 R09: 0000000000000000 [10392.849692] R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000008 [10392.849693] R13: 0000000000000000 R14: 0000000000000001 R15: 0000000000000000 [10392.849699] </TASK> [10470.858708] rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: [10470.859282] rcu: 0-....: (1 GPs behind) idle=7cf4/1/0x4000000000000000 softirq=41150/41152 fqs=2103308 [10470.860040] rcu: 2-....: (1 GPs behind) idle=926c/1/0x4000000000000000 softirq=41281/41282 fqs=2103308 [10470.860792] rcu: (detected by 1, t=9312144 jiffies, g=77753, q=157 ncpus=4) [10470.861392] Sending NMI from CPU 1 to CPUs 0: [10470.861802] NMI backtrace for cpu 0 [10470.861810] CPU: 0 PID: 5627 Comm: ncat Not tainted 6.8.0-rc2-virtme #1 [10470.861814] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [10470.861816] RIP: 0010:queued_spin_lock_slowpath+0x36/0x310 [10470.861828] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 <eb> e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 [10470.861831] RSP: 0000:ffffbef24018f688 EFLAGS: 00000202 [10470.861834] RAX: 0000000000000001 RBX: ffff9a9dc2ac0c98 RCX: 0000000000000001 [10470.861836] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9a9dc2ac0c98 [10470.861838] RBP: ffff9a9dc2ac0c98 R08: ffffbef24018f448 R09: 0000000000000020 [10470.861839] R10: ffffbef24018f538 R11: 0000000000000001 R12: 0000000000000000 [10470.861841] R13: ffff9a9dc2ac0c00 R14: ffff9a9dc2ac0c98 R15: ffff9a9dc1d5054c [10470.861845] FS: 00007fd2c23d9740(0000) GS:ffff9a9dfec00000(0000) knlGS:0000000000000000 [10470.861847] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [10470.861849] CR2: 0000559104c93000 CR3: 0000000003d4e005 CR4: 0000000000770ef0 [10470.861851] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [10470.861852] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [10470.861854] PKRU: 55555554 [10470.861855] Call Trace: [10470.861858] <NMI> [10470.861864] ? nmi_cpu_backtrace+0xa2/0x110 [10470.861872] ? nmi_cpu_backtrace_handler+0x11/0x20 [10470.861879] ? nmi_handle+0xd0/0x270 [10470.861884] ? default_do_nmi+0x41/0x100 [10470.861891] ? exc_nmi+0xf3/0x120 [10470.861894] ? end_repeat_nmi+0xf/0x60 [10470.861905] ? queued_spin_lock_slowpath+0x36/0x310 [10470.861910] ? queued_spin_lock_slowpath+0x36/0x310 [10470.861914] ? queued_spin_lock_slowpath+0x36/0x310 [10470.861918] </NMI> [10470.861919] <TASK> [10470.861920] do_raw_spin_lock+0x9b/0xa0 [10470.861928] tcp_v4_rcv+0xdb5/0xef0 [10470.861938] ip_protocol_deliver_rcu+0x4b/0x1e0 [10470.861945] ip_local_deliver_finish+0xaf/0x180 [10470.861948] __netif_receive_skb_one_core+0x8b/0xa0 [10470.861956] netif_receive_skb+0x87/0x2c0 [10470.861962] tcf_mirred_to_dev+0x1f6/0x410 [act_mirred] [10470.861971] tcf_mirred_act+0xee/0x560 [act_mirred] [10470.861978] ? tcf_skbedit_act+0x1d6/0x410 [act_skbedit] [10470.861983] ? lock_acquire+0xc1/0x2b0 [10470.861988] tcf_action_exec+0xac/0x230 [10470.861996] fl_classify+0x20e/0x260 [cls_flower] [10470.862009] ? __bfs+0xf3/0x240 [10470.862013] ? check_irq_usage+0x19c/0xc40 [10470.862016] ? __lock_acquire+0xa6e/0x1750 [10470.862021] ? lock_acquire+0x179/0x2b0 [10470.862024] ? check_path.constprop.0+0x24/0x50 [10470.862027] ? check_noncircular+0x81/0x150 [10470.862030] ? lock_acquire+0x179/0x2b0 [10470.862033] ? is_bpf_text_address+0x71/0x100 [10470.862037] ? lock_release+0x1d3/0x270 [10470.862042] ? __lock_acquire+0xa6e/0x1750 [10470.862048] tcf_classify+0x17a/0x540 [10470.862055] tc_run+0x9f/0x150 [10470.862063] __dev_queue_xmit+0x311/0x1180 [10470.862067] ? mark_held_locks+0x49/0x80 [10470.862070] ? eth_header+0x2a/0xc0 [10470.862076] ip_finish_output2+0x22d/0xa20 [10470.862083] ? __ip_queue_xmit+0x1d5/0x6f0 [10470.862086] __ip_queue_xmit+0x1d5/0x6f0 [10470.862091] __tcp_transmit_skb+0xb65/0xd30 [10470.862097] ? __alloc_skb+0xdc/0x1a0 [10470.862102] ? __pfx_tcp_delack_timer+0x10/0x10 [10470.862104] tcp_delack_timer_handler+0xa3/0xf0 [10470.862107] tcp_delack_timer+0x5a/0x140 [10470.862110] ? __pfx_tcp_delack_timer+0x10/0x10 [10470.862112] call_timer_fn+0xa5/0x220 [10470.862118] __run_timers.part.0+0x1f8/0x2c0 [10470.862122] ? kvm_clock_get_cycles+0x18/0x30 [10470.862128] run_timer_softirq+0x2a/0x50 [10470.862130] __do_softirq+0xc8/0x37a [10470.862135] irq_exit_rcu+0x7f/0xa0 [10470.862140] sysvec_apic_timer_interrupt+0x41/0x90 [10470.862143] asm_sysvec_apic_timer_interrupt+0x1a/0x20 [10470.862146] RIP: 0033:0x7fd2c2613700 [10470.862149] Code: ff ff 64 89 02 eb bd 0f 1f 00 f3 0f 1e fa 41 89 ca 64 8b 04 25 18 00 00 00 85 c0 75 1d 45 31 c9 45 31 c0 b8 2c 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 68 c3 0f 1f 80 00 00 00 00 41 54 48 83 ec 20 [10470.862151] RSP: 002b:00007ffe6ba49238 EFLAGS: 00000246 [10470.862153] RAX: 0000000000002000 RBX: 0000000000002000 RCX: 00007fd2c2613700 [10470.862155] RDX: 0000000000002000 RSI: 0000559104c92780 RDI: 0000000000000005 [10470.862156] RBP: 0000559104b589e0 R08: 0000000000000000 R09: 0000000000000000 [10470.862157] R10: 0000000000000000 R11: 0000000000000246 R12: 0000559104bbccf0 [10470.862159] R13: 0000559104bbcb10 R14: 0000559104bbcb10 R15: 0000559104b589e0 [10470.862164] </TASK> [10470.862782] Sending NMI from CPU 1 to CPUs 2: [10471.070714] NMI backtrace for cpu 2 [10471.070721] CPU: 2 PID: 5626 Comm: ncat Not tainted 6.8.0-rc2-virtme #1 [10471.070725] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [10471.070727] RIP: 0010:queued_spin_lock_slowpath+0x36/0x310 [10471.070737] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 <eb> e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 [10471.070740] RSP: 0018:ffffbef2401077a8 EFLAGS: 00000202 [10471.070743] RAX: 0000000000000001 RBX: ffff9a9dc2ac0098 RCX: 0000000000000001 [10471.070745] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9a9dc2ac0098 [10471.070746] RBP: ffff9a9dc2ac0098 R08: 0000000000000008 R09: ffffffff96d9d1a8 [10471.070748] R10: 0000000000000000 R11: 0000000000000001 R12: 0000000000000000 [10471.070749] R13: ffff9a9dc2ac0000 R14: ffff9a9dc2ac0098 R15: ffff9a9dc23ea94c [10471.070754] FS: 00007fa251394740(0000) GS:ffff9a9dfed00000(0000) knlGS:0000000000000000 [10471.070756] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [10471.070757] CR2: 00005653cd1c7340 CR3: 000000000263c001 CR4: 0000000000770ef0 [10471.070759] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [10471.070760] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [10471.070761] PKRU: 55555554 [10471.070763] Call Trace: [10471.070766] <NMI> [10471.070770] ? nmi_cpu_backtrace+0xa2/0x110 [10471.070778] ? nmi_cpu_backtrace_handler+0x11/0x20 [10471.070784] ? nmi_handle+0xd0/0x270 [10471.070790] ? default_do_nmi+0x41/0x100 [10471.070795] ? exc_nmi+0xf3/0x120 [10471.070798] ? end_repeat_nmi+0xf/0x60 [10471.070808] ? queued_spin_lock_slowpath+0x36/0x310 [10471.070812] ? queued_spin_lock_slowpath+0x36/0x310 [10471.070817] ? queued_spin_lock_slowpath+0x36/0x310 [10471.070820] </NMI> [10471.070821] <TASK> [10471.070823] do_raw_spin_lock+0x9b/0xa0 [10471.070828] tcp_v4_rcv+0xdb5/0xef0 [10471.070837] ip_protocol_deliver_rcu+0x4b/0x1e0 [10471.070843] ip_local_deliver_finish+0xaf/0x180 [10471.070845] __netif_receive_skb_one_core+0x8b/0xa0 [10471.070852] netif_receive_skb+0x87/0x2c0 [10471.070857] tcf_mirred_to_dev+0x1f6/0x410 [act_mirred] [10471.070866] tcf_mirred_act+0xee/0x560 [act_mirred] [10471.070873] ? tcf_skbedit_act+0x1d6/0x410 [act_skbedit] [10471.070880] tcf_action_exec+0xac/0x230 [10471.070888] fl_classify+0x20e/0x260 [cls_flower] [10471.070899] ? kernel_text_address+0xce/0xe0 [10471.070904] ? __kernel_text_address+0x12/0x40 [10471.070906] ? unwind_get_return_address+0x23/0x40 [10471.070910] ? __pfx_stack_trace_consume_entry+0x10/0x10 [10471.070915] ? arch_stack_walk+0xa2/0x100 [10471.070922] ? stack_trace_save+0x4f/0x80 [10471.070926] ? save_trace+0x3f/0x3b0 [10471.070931] ? __lock_acquire+0xa6e/0x1750 [10471.070938] tcf_classify+0x17a/0x540 [10471.070945] tc_run+0x9f/0x150 [10471.070951] __dev_queue_xmit+0x311/0x1180 [10471.070955] ? mark_held_locks+0x49/0x80 [10471.070958] ? eth_header+0x2a/0xc0 [10471.070963] ip_finish_output2+0x22d/0xa20 [10471.070967] ? ip_skb_dst_mtu+0x51/0x1c0 [10471.070971] ? __ip_queue_xmit+0x1d5/0x6f0 [10471.070974] __ip_queue_xmit+0x1d5/0x6f0 [10471.070979] __tcp_transmit_skb+0xb65/0xd30 [10471.070985] ? __alloc_skb+0xdc/0x1a0 [10471.070990] release_sock+0x4f/0xb0 [10471.070994] inet_shutdown+0xa8/0x100 [10471.070999] __sys_shutdown+0x45/0x90 [10471.071004] __x64_sys_shutdown+0x14/0x20 [10471.071006] do_syscall_64+0xc9/0x1e0 [10471.071010] entry_SYSCALL_64_after_hwframe+0x6f/0x77 [10471.071013] RIP: 0033:0x7fa2514bebeb [10471.071017] Code: 73 01 c3 48 8b 0d 15 92 1b 00 f7 d8 64 89 01 48 83 c8 ff c3 66 2e 0f 1f 84 00 00 00 00 00 90 f3 0f 1e fa b8 30 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d e5 91 1b 00 f7 d8 64 89 01 48 [10471.071019] RSP: 002b:00007ffe4efd4718 EFLAGS: 00000246 ORIG_RAX: 0000000000000030 [10471.071021] RAX: ffffffffffffffda RBX: 0000000000000008 RCX: 00007fa2514bebeb [10471.071023] RDX: 00005653cd662420 RSI: 0000000000000001 RDI: 0000000000000008 [10471.071024] RBP: 0000000000000008 R08: 0000000000000000 R09: 0000000000000000 [10471.071025] R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000008 [10471.071026] R13: 0000000000000000 R14: 0000000000000001 R15: 0000000000000000 [10471.071031] </TASK> [10549.116708] rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: [10549.117289] rcu: 0-....: (1 GPs behind) idle=7cf4/1/0x4000000000000000 softirq=41150/41152 fqs=2118584 [10549.118078] rcu: 2-....: (1 GPs behind) idle=926c/1/0x4000000000000000 softirq=41281/41282 fqs=2118584 [10549.118864] rcu: (detected by 3, t=9390402 jiffies, g=77753, q=157 ncpus=4) [10549.119466] Sending NMI from CPU 3 to CPUs 0: [10549.124794] NMI backtrace for cpu 0 [10549.124814] CPU: 0 PID: 5627 Comm: ncat Not tainted 6.8.0-rc2-virtme #1 [10549.124818] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [10549.124821] RIP: 0010:queued_spin_lock_slowpath+0x36/0x310 [10549.124833] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 <eb> e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 [10549.124835] RSP: 0000:ffffbef24018f688 EFLAGS: 00000202 [10549.124838] RAX: 0000000000000001 RBX: ffff9a9dc2ac0c98 RCX: 0000000000000001 [10549.124840] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9a9dc2ac0c98 [10549.124842] RBP: ffff9a9dc2ac0c98 R08: ffffbef24018f448 R09: 0000000000000020 [10549.124843] R10: ffffbef24018f538 R11: 0000000000000001 R12: 0000000000000000 [10549.124845] R13: ffff9a9dc2ac0c00 R14: ffff9a9dc2ac0c98 R15: ffff9a9dc1d5054c [10549.124850] FS: 00007fd2c23d9740(0000) GS:ffff9a9dfec00000(0000) knlGS:0000000000000000 [10549.124852] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [10549.124854] CR2: 0000559104c93000 CR3: 0000000003d4e005 CR4: 0000000000770ef0 [10549.124855] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [10549.124857] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [10549.124859] PKRU: 55555554 [10549.124860] Call Trace: [10549.124864] <NMI> [10549.124869] ? nmi_cpu_backtrace+0xa2/0x110 [10549.124877] ? nmi_cpu_backtrace_handler+0x11/0x20 [10549.124884] ? nmi_handle+0xd0/0x270 [10549.124888] ? default_do_nmi+0x41/0x100 [10549.124894] ? exc_nmi+0xf3/0x120 [10549.124897] ? end_repeat_nmi+0xf/0x60 [10549.124907] ? queued_spin_lock_slowpath+0x36/0x310 [10549.124911] ? queued_spin_lock_slowpath+0x36/0x310 [10549.124916] ? queued_spin_lock_slowpath+0x36/0x310 [10549.124919] </NMI> [10549.124920] <TASK> [10549.124922] do_raw_spin_lock+0x9b/0xa0 [10549.124929] tcp_v4_rcv+0xdb5/0xef0 [10549.124939] ip_protocol_deliver_rcu+0x4b/0x1e0 [10549.124945] ip_local_deliver_finish+0xaf/0x180 [10549.124948] __netif_receive_skb_one_core+0x8b/0xa0 [10549.124956] netif_receive_skb+0x87/0x2c0 [10549.124962] tcf_mirred_to_dev+0x1f6/0x410 [act_mirred] [10549.124972] tcf_mirred_act+0xee/0x560 [act_mirred] [10549.124978] ? tcf_skbedit_act+0x1d6/0x410 [act_skbedit] [10549.124984] ? lock_acquire+0xc1/0x2b0 [10549.124988] tcf_action_exec+0xac/0x230 [10549.124996] fl_classify+0x20e/0x260 [cls_flower] [10549.125008] ? __bfs+0xf3/0x240 [10549.125013] ? check_irq_usage+0x19c/0xc40 [10549.125015] ? __lock_acquire+0xa6e/0x1750 [10549.125019] ? lock_acquire+0x179/0x2b0 [10549.125023] ? check_path.constprop.0+0x24/0x50 [10549.125026] ? check_noncircular+0x81/0x150 [10549.125029] ? lock_acquire+0x179/0x2b0 [10549.125032] ? is_bpf_text_address+0x71/0x100 [10549.125036] ? lock_release+0x1d3/0x270 [10549.125040] ? __lock_acquire+0xa6e/0x1750 [10549.125046] tcf_classify+0x17a/0x540 [10549.125052] tc_run+0x9f/0x150 [10549.125060] __dev_queue_xmit+0x311/0x1180 [10549.125063] ? mark_held_locks+0x49/0x80 [10549.125067] ? eth_header+0x2a/0xc0 [10549.125073] ip_finish_output2+0x22d/0xa20 [10549.125080] ? __ip_queue_xmit+0x1d5/0x6f0 [10549.125083] __ip_queue_xmit+0x1d5/0x6f0 [10549.125088] __tcp_transmit_skb+0xb65/0xd30 [10549.125095] ? __alloc_skb+0xdc/0x1a0 [10549.125101] ? __pfx_tcp_delack_timer+0x10/0x10 [10549.125104] tcp_delack_timer_handler+0xa3/0xf0 [10549.125107] tcp_delack_timer+0x5a/0x140 [10549.125109] ? __pfx_tcp_delack_timer+0x10/0x10 [10549.125112] call_timer_fn+0xa5/0x220 [10549.125117] __run_timers.part.0+0x1f8/0x2c0 [10549.125122] ? kvm_clock_get_cycles+0x18/0x30 [10549.125129] run_timer_softirq+0x2a/0x50 [10549.125131] __do_softirq+0xc8/0x37a [10549.125137] irq_exit_rcu+0x7f/0xa0 [10549.125142] sysvec_apic_timer_interrupt+0x41/0x90 [10549.125146] asm_sysvec_apic_timer_interrupt+0x1a/0x20 [10549.125149] RIP: 0033:0x7fd2c2613700 [10549.125154] Code: ff ff 64 89 02 eb bd 0f 1f 00 f3 0f 1e fa 41 89 ca 64 8b 04 25 18 00 00 00 85 c0 75 1d 45 31 c9 45 31 c0 b8 2c 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 68 c3 0f 1f 80 00 00 00 00 41 54 48 83 ec 20 [10549.125156] RSP: 002b:00007ffe6ba49238 EFLAGS: 00000246 [10549.125158] RAX: 0000000000002000 RBX: 0000000000002000 RCX: 00007fd2c2613700 [10549.125160] RDX: 0000000000002000 RSI: 0000559104c92780 RDI: 0000000000000005 [10549.125161] RBP: 0000559104b589e0 R08: 0000000000000000 R09: 0000000000000000 [10549.125163] R10: 0000000000000000 R11: 0000000000000246 R12: 0000559104bbccf0 [10549.125164] R13: 0000559104bbcb10 R14: 0000559104bbcb10 R15: 0000559104b589e0 [10549.125171] </TASK> [10549.125837] Sending NMI from CPU 3 to CPUs 2: [10549.368708] NMI backtrace for cpu 2 [10549.368718] CPU: 2 PID: 5626 Comm: ncat Not tainted 6.8.0-rc2-virtme #1 [10549.368739] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [10549.368742] RIP: 0010:queued_spin_lock_slowpath+0x36/0x310 [10549.368754] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 <eb> e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 [10549.368757] RSP: 0018:ffffbef2401077a8 EFLAGS: 00000202 [10549.368760] RAX: 0000000000000001 RBX: ffff9a9dc2ac0098 RCX: 0000000000000001 [10549.368762] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9a9dc2ac0098 [10549.368764] RBP: ffff9a9dc2ac0098 R08: 0000000000000008 R09: ffffffff96d9d1a8 [10549.368765] R10: 0000000000000000 R11: 0000000000000001 R12: 0000000000000000 [10549.368766] R13: ffff9a9dc2ac0000 R14: ffff9a9dc2ac0098 R15: ffff9a9dc23ea94c [10549.368772] FS: 00007fa251394740(0000) GS:ffff9a9dfed00000(0000) knlGS:0000000000000000 [10549.368774] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [10549.368776] CR2: 00005653cd1c7340 CR3: 000000000263c001 CR4: 0000000000770ef0 [10549.368778] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [10549.368779] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [10549.368781] PKRU: 55555554 [10549.368782] Call Trace: [10549.368786] <NMI> [10549.368792] ? nmi_cpu_backtrace+0xa2/0x110 [10549.368800] ? nmi_cpu_backtrace_handler+0x11/0x20 [10549.368808] ? nmi_handle+0xd0/0x270 [10549.368813] ? default_do_nmi+0x41/0x100 [10549.368819] ? exc_nmi+0xf3/0x120 [10549.368822] ? end_repeat_nmi+0xf/0x60 [10549.368833] ? queued_spin_lock_slowpath+0x36/0x310 [10549.368837] ? queued_spin_lock_slowpath+0x36/0x310 [10549.368841] ? queued_spin_lock_slowpath+0x36/0x310 [10549.368845] </NMI> [10549.368846] <TASK> [10549.368848] do_raw_spin_lock+0x9b/0xa0 [10549.368856] tcp_v4_rcv+0xdb5/0xef0 [10549.368867] ip_protocol_deliver_rcu+0x4b/0x1e0 [10549.368874] ip_local_deliver_finish+0xaf/0x180 [10549.368876] __netif_receive_skb_one_core+0x8b/0xa0 [10549.368885] netif_receive_skb+0x87/0x2c0 [10549.368892] tcf_mirred_to_dev+0x1f6/0x410 [act_mirred] [10549.368901] tcf_mirred_act+0xee/0x560 [act_mirred] [10549.368908] ? tcf_skbedit_act+0x1d6/0x410 [act_skbedit] [10549.368916] tcf_action_exec+0xac/0x230 [10549.368924] fl_classify+0x20e/0x260 [cls_flower] [10549.368936] ? kernel_text_address+0xce/0xe0 [10549.368942] ? __kernel_text_address+0x12/0x40 [10549.368944] ? unwind_get_return_address+0x23/0x40 [10549.368949] ? __pfx_stack_trace_consume_entry+0x10/0x10 [10549.368954] ? arch_stack_walk+0xa2/0x100 [10549.368961] ? stack_trace_save+0x4f/0x80 [10549.368965] ? save_trace+0x3f/0x3b0 [10549.368970] ? __lock_acquire+0xa6e/0x1750 [10549.368976] tcf_classify+0x17a/0x540 [10549.368984] tc_run+0x9f/0x150 [10549.368992] __dev_queue_xmit+0x311/0x1180 [10549.368995] ? mark_held_locks+0x49/0x80 [10549.368998] ? eth_header+0x2a/0xc0 [10549.369004] ip_finish_output2+0x22d/0xa20 [10549.369009] ? ip_skb_dst_mtu+0x51/0x1c0 [10549.369013] ? __ip_queue_xmit+0x1d5/0x6f0 [10549.369016] __ip_queue_xmit+0x1d5/0x6f0 [10549.369021] __tcp_transmit_skb+0xb65/0xd30 [10549.369028] ? __alloc_skb+0xdc/0x1a0 [10549.369034] release_sock+0x4f/0xb0 [10549.369039] inet_shutdown+0xa8/0x100 [10549.369044] __sys_shutdown+0x45/0x90 [10549.369049] __x64_sys_shutdown+0x14/0x20 [10549.369052] do_syscall_64+0xc9/0x1e0 [10549.369056] entry_SYSCALL_64_after_hwframe+0x6f/0x77 [10549.369059] RIP: 0033:0x7fa2514bebeb [10549.369064] Code: 73 01 c3 48 8b 0d 15 92 1b 00 f7 d8 64 89 01 48 83 c8 ff c3 66 2e 0f 1f 84 00 00 00 00 00 90 f3 0f 1e fa b8 30 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d e5 91 1b 00 f7 d8 64 89 01 48 [10549.369067] RSP: 002b:00007ffe4efd4718 EFLAGS: 00000246 ORIG_RAX: 0000000000000030 [10549.369070] RAX: ffffffffffffffda RBX: 0000000000000008 RCX: 00007fa2514bebeb [10549.369072] RDX: 00005653cd662420 RSI: 0000000000000001 RDI: 0000000000000008 [10549.369073] RBP: 0000000000000008 R08: 0000000000000000 R09: 0000000000000000 [10549.369075] R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000008 [10549.369076] R13: 0000000000000000 R14: 0000000000000001 R15: 0000000000000000 [10549.369082] </TASK> [10627.389731] rcu: INFO: rcu_preempt self-detected stall on CPU [10627.390267] rcu: 0-....: (1 GPs behind) idle=7cf4/1/0x4000000000000000 softirq=41150/41152 fqs=2134293 [10627.391047] rcu: (t=9468673 jiffies g=77753 q=157 ncpus=4) [10627.391528] CPU: 0 PID: 5627 Comm: ncat Not tainted 6.8.0-rc2-virtme #1 [10627.392076] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [10627.393046] RIP: 0010:queued_spin_lock_slowpath+0x36/0x310 [10627.393526] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 <eb> e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 [10627.395078] RSP: 0000:ffffbef24018f688 EFLAGS: 00000202 [10627.395520] RAX: 0000000000000001 RBX: ffff9a9dc2ac0c98 RCX: 0000000000000001 [10627.396105] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9a9dc2ac0c98 [10627.396694] RBP: ffff9a9dc2ac0c98 R08: ffffbef24018f448 R09: 0000000000000020 [10627.397315] R10: ffffbef24018f538 R11: 0000000000000001 R12: 0000000000000000 [10627.397927] R13: ffff9a9dc2ac0c00 R14: ffff9a9dc2ac0c98 R15: ffff9a9dc1d5054c [10627.428636] FS: 00007fd2c23d9740(0000) GS:ffff9a9dfec00000(0000) knlGS:0000000000000000 [10627.429325] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [10627.429822] CR2: 0000559104c93000 CR3: 0000000003d4e005 CR4: 0000000000770ef0 [10627.430419] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [10627.431022] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [10627.431655] PKRU: 55555554 [10627.431909] Call Trace: [10627.432148] <IRQ> [10627.432348] ? rcu_dump_cpu_stacks+0xfa/0x180 [10627.432742] ? rcu_sched_clock_irq+0x7bb/0x13d0 [10627.433151] ? lock_is_held_type+0x9f/0x120 [10627.433548] ? lock_is_held_type+0x9f/0x120 [10627.433913] ? update_process_times+0x5f/0x90 [10627.434290] ? __pfx_tick_nohz_highres_handler+0x10/0x10 [10627.434748] ? tick_sched_handle+0x33/0x50 [10627.435099] ? tick_nohz_highres_handler+0x71/0x90 [10627.435520] ? __hrtimer_run_queues+0x86/0x390 [10627.435908] ? hrtimer_interrupt+0x100/0x240 [10627.436292] ? __sysvec_apic_timer_interrupt+0x66/0x1a0 [10627.436738] ? sysvec_apic_timer_interrupt+0x73/0x90 [10627.437190] </IRQ> [10627.437394] <TASK> [10627.437595] ? asm_sysvec_apic_timer_interrupt+0x1a/0x20 [10627.438082] ? queued_spin_lock_slowpath+0x36/0x310 [10627.458622] do_raw_spin_lock+0x9b/0xa0 [10627.458972] tcp_v4_rcv+0xdb5/0xef0 [10627.459299] ip_protocol_deliver_rcu+0x4b/0x1e0 [10627.459705] ip_local_deliver_finish+0xaf/0x180 [10627.460104] __netif_receive_skb_one_core+0x8b/0xa0 [10627.460526] netif_receive_skb+0x87/0x2c0 [10627.460897] tcf_mirred_to_dev+0x1f6/0x410 [act_mirred] [10627.461351] tcf_mirred_act+0xee/0x560 [act_mirred] [10627.461773] ? tcf_skbedit_act+0x1d6/0x410 [act_skbedit] [10627.462242] ? lock_acquire+0xc1/0x2b0 [10627.462592] tcf_action_exec+0xac/0x230 [10627.462931] fl_classify+0x20e/0x260 [cls_flower] [10627.463358] ? __bfs+0xf3/0x240 [10627.463638] ? check_irq_usage+0x19c/0xc40 [10627.463985] ? __lock_acquire+0xa6e/0x1750 [10627.464335] ? lock_acquire+0x179/0x2b0 [10627.464670] ? check_path.constprop.0+0x24/0x50 [10627.465053] ? check_noncircular+0x81/0x150 [10627.465409] ? lock_acquire+0x179/0x2b0 [10627.465740] ? is_bpf_text_address+0x71/0x100 [10627.466121] ? lock_release+0x1d3/0x270 [10627.466478] ? __lock_acquire+0xa6e/0x1750 [10627.466849] tcf_classify+0x17a/0x540 [10627.467186] tc_run+0x9f/0x150 [10627.467473] __dev_queue_xmit+0x311/0x1180 [10627.467837] ? mark_held_locks+0x49/0x80 [10627.468188] ? eth_header+0x2a/0xc0 [10627.482693] ip_finish_output2+0x22d/0xa20 [10627.483083] ? __ip_queue_xmit+0x1d5/0x6f0 [10627.483468] __ip_queue_xmit+0x1d5/0x6f0 [10627.483825] __tcp_transmit_skb+0xb65/0xd30 [10627.484217] ? __alloc_skb+0xdc/0x1a0 [10627.484551] ? __pfx_tcp_delack_timer+0x10/0x10 [10627.484955] tcp_delack_timer_handler+0xa3/0xf0 [10627.485352] tcp_delack_timer+0x5a/0x140 [10627.485696] ? __pfx_tcp_delack_timer+0x10/0x10 [10627.486086] call_timer_fn+0xa5/0x220 [10627.486413] __run_timers.part.0+0x1f8/0x2c0 [10627.486789] ? kvm_clock_get_cycles+0x18/0x30 [10627.487173] run_timer_softirq+0x2a/0x50 [10627.487510] __do_softirq+0xc8/0x37a [10627.487826] irq_exit_rcu+0x7f/0xa0 [10627.488135] sysvec_apic_timer_interrupt+0x41/0x90 [10627.488562] asm_sysvec_apic_timer_interrupt+0x1a/0x20 [10627.488993] RIP: 0033:0x7fd2c2613700 [10627.489321] Code: ff ff 64 89 02 eb bd 0f 1f 00 f3 0f 1e fa 41 89 ca 64 8b 04 25 18 00 00 00 85 c0 75 1d 45 31 c9 45 31 c0 b8 2c 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 68 c3 0f 1f 80 00 00 00 00 41 54 48 83 ec 20 [10627.490866] RSP: 002b:00007ffe6ba49238 EFLAGS: 00000246 [10627.491356] RAX: 0000000000002000 RBX: 0000000000002000 RCX: 00007fd2c2613700 [10627.491996] RDX: 0000000000002000 RSI: 0000559104c92780 RDI: 0000000000000005 [10627.492598] RBP: 0000559104b589e0 R08: 0000000000000000 R09: 0000000000000000 [10627.493192] R10: 0000000000000000 R11: 0000000000000246 R12: 0000559104bbccf0 [10627.493785] R13: 0000559104bbcb10 R14: 0000559104bbcb10 R15: 0000559104b589e0 [10627.494389] </TASK> [10627.494586] Sending NMI from CPU 0 to CPUs 2: [10627.494972] NMI backtrace for cpu 2 [10627.494978] CPU: 2 PID: 5626 Comm: ncat Not tainted 6.8.0-rc2-virtme #1 [10627.494981] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [10627.494984] RIP: 0010:queued_spin_lock_slowpath+0x36/0x310 [10627.494995] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 <eb> e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 [10627.494998] RSP: 0018:ffffbef2401077a8 EFLAGS: 00000202 [10627.495001] RAX: 0000000000000001 RBX: ffff9a9dc2ac0098 RCX: 0000000000000001 [10627.495003] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9a9dc2ac0098 [10627.495004] RBP: ffff9a9dc2ac0098 R08: 0000000000000008 R09: ffffffff96d9d1a8 [10627.495006] R10: 0000000000000000 R11: 0000000000000001 R12: 0000000000000000 [10627.495007] R13: ffff9a9dc2ac0000 R14: ffff9a9dc2ac0098 R15: ffff9a9dc23ea94c [10627.495011] FS: 00007fa251394740(0000) GS:ffff9a9dfed00000(0000) knlGS:0000000000000000 [10627.495013] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [10627.495015] CR2: 00005653cd1c7340 CR3: 000000000263c001 CR4: 0000000000770ef0 [10627.495016] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [10627.495018] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [10627.495019] PKRU: 55555554 [10627.495020] Call Trace: [10627.495024] <NMI> [10627.495027] ? nmi_cpu_backtrace+0xa2/0x110 [10627.495035] ? nmi_cpu_backtrace_handler+0x11/0x20 [10627.495042] ? nmi_handle+0xd0/0x270 [10627.495047] ? default_do_nmi+0x41/0x100 [10627.495053] ? exc_nmi+0xf3/0x120 [10627.495057] ? end_repeat_nmi+0xf/0x60 [10627.495067] ? queued_spin_lock_slowpath+0x36/0x310 [10627.495071] ? queued_spin_lock_slowpath+0x36/0x310 [10627.495076] ? queued_spin_lock_slowpath+0x36/0x310 [10627.495080] </NMI> [10627.495081] <TASK> [10627.495083] do_raw_spin_lock+0x9b/0xa0 [10627.495090] tcp_v4_rcv+0xdb5/0xef0 [10627.495100] ip_protocol_deliver_rcu+0x4b/0x1e0 [10627.495107] ip_local_deliver_finish+0xaf/0x180 [10627.495110] __netif_receive_skb_one_core+0x8b/0xa0 [10627.495119] netif_receive_skb+0x87/0x2c0 [10627.495124] tcf_mirred_to_dev+0x1f6/0x410 [act_mirred] [10627.495134] tcf_mirred_act+0xee/0x560 [act_mirred] [10627.495141] ? tcf_skbedit_act+0x1d6/0x410 [act_skbedit] [10627.495149] tcf_action_exec+0xac/0x230 [10627.495157] fl_classify+0x20e/0x260 [cls_flower] [10627.495169] ? kernel_text_address+0xce/0xe0 [10627.495173] ? __kernel_text_address+0x12/0x40 [10627.495176] ? unwind_get_return_address+0x23/0x40 [10627.495179] ? __pfx_stack_trace_consume_entry+0x10/0x10 [10627.495184] ? arch_stack_walk+0xa2/0x100 [10627.495191] ? stack_trace_save+0x4f/0x80 [10627.495195] ? save_trace+0x3f/0x3b0 [10627.495200] ? __lock_acquire+0xa6e/0x1750 [10627.495205] tcf_classify+0x17a/0x540 [10627.495211] tc_run+0x9f/0x150 [10627.495217] __dev_queue_xmit+0x311/0x1180 [10627.495221] ? mark_held_locks+0x49/0x80 [10627.495224] ? eth_header+0x2a/0xc0 [10627.495231] ip_finish_output2+0x22d/0xa20 [10627.495235] ? ip_skb_dst_mtu+0x51/0x1c0 [10627.495240] ? __ip_queue_xmit+0x1d5/0x6f0 [10627.495242] __ip_queue_xmit+0x1d5/0x6f0 [10627.495247] __tcp_transmit_skb+0xb65/0xd30 [10627.495255] ? __alloc_skb+0xdc/0x1a0 [10627.495261] release_sock+0x4f/0xb0 [10627.495265] inet_shutdown+0xa8/0x100 [10627.495271] __sys_shutdown+0x45/0x90 [10627.495277] __x64_sys_shutdown+0x14/0x20 [10627.495280] do_syscall_64+0xc9/0x1e0 [10627.495284] entry_SYSCALL_64_after_hwframe+0x6f/0x77 [10627.495287] RIP: 0033:0x7fa2514bebeb [10627.495292] Code: 73 01 c3 48 8b 0d 15 92 1b 00 f7 d8 64 89 01 48 83 c8 ff c3 66 2e 0f 1f 84 00 00 00 00 00 90 f3 0f 1e fa b8 30 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d e5 91 1b 00 f7 d8 64 89 01 48 [10627.495294] RSP: 002b:00007ffe4efd4718 EFLAGS: 00000246 ORIG_RAX: 0000000000000030 [10627.495297] RAX: ffffffffffffffda RBX: 0000000000000008 RCX: 00007fa2514bebeb [10627.495299] RDX: 00005653cd662420 RSI: 0000000000000001 RDI: 0000000000000008 [10627.495300] RBP: 0000000000000008 R08: 0000000000000000 R09: 0000000000000000 [10627.495301] R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000008 [10627.495302] R13: 0000000000000000 R14: 0000000000000001 R15: 0000000000000000 [10627.495308] </TASK> [10705.495690] rcu: INFO: rcu_preempt self-detected stall on CPU [10705.496032] rcu: 2-....: (1 GPs behind) idle=926c/1/0x4000000000000000 softirq=41281/41282 fqs=2151721 [10705.496518] rcu: (t=9546779 jiffies g=77753 q=157 ncpus=4) [10705.496810] Sending NMI from CPU 2 to CPUs 0: [10705.497055] NMI backtrace for cpu 0 [10705.497067] CPU: 0 PID: 5627 Comm: ncat Not tainted 6.8.0-rc2-virtme #1 [10705.497069] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [10705.497071] RIP: 0010:queued_spin_lock_slowpath+0x36/0x310 [10705.497082] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc cc cc f3 90 <eb> e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00 [10705.497083] RSP: 0000:ffffbef24018f688 EFLAGS: 00000202 [10705.497085] RAX: 0000000000000001 RBX: ffff9a9dc2ac0c98 RCX: 0000000000000001 [10705.497087] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9a9dc2ac0c98 [10705.497088] RBP: ffff9a9dc2ac0c98 R08: ffffbef24018f448 R09: 0000000000000020 [10705.497089] R10: ffffbef24018f538 R11: 0000000000000001 R12: 0000000000000000 [10705.497089] R13: ffff9a9dc2ac0c00 R14: ffff9a9dc2ac0c98 R15: ffff9a9dc1d5054c [10705.497092] FS: 00007fd2c23d9740(0000) GS:ffff9a9dfec00000(0000) knlGS:0000000000000000 [10705.497094] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [10705.497095] CR2: 0000559104c93000 CR3: 0000000003d4e005 CR4: 0000000000770ef0 [10705.497096] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [10705.497097] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [10705.497098] PKRU: 55555554 [10705.497098] Call Trace: [10705.497101] <NMI> [10705.497105] ? nmi_cpu_backtrace+0xa2/0x110 [10705.497110] ? nmi_cpu_backtrace_handler+0x11/0x20 [10705.497117] ? nmi_handle+0xd0/0x270 [10705.497120] ? default_do_nmi+0x41/0x100 [10705.497125] ? exc_nmi+0xf3/0x120 [10705.497127] ? end_repeat_nmi+0xf/0x60 [10705.497135] ? queued_spin_lock_slowpath+0x36/0x310 [10705.497138] ? queued_spin_lock_slowpath+0x36/0x310 [10705.497141] ? queued_spin_lock_slowpath+0x36/0x310 [10705.497143] </NMI> [10705.497144] <TASK> [10705.497146] do_raw_spin_lock+0x9b/0xa0 [10705.497152] tcp_v4_rcv+0xdb5/0xef0 [10705.497159] ip_protocol_deliver_rcu+0x4b/0x1e0 [10705.497164] ip_local_deliver_finish+0xaf/0x180 [10705.497167] __netif_receive_skb_one_core+0x8b/0xa0 [10705.497172] netif_receive_skb+0x87/0x2c0 [10705.497177] tcf_mirred_to_dev+0x1f6/0x410 [act_mirred] [10705.497184] tcf_mirred_act+0xee/0x560 [act_mirred] [10705.497189] ? tcf_skbedit_act+0x1d6/0x410 [act_skbedit] [10705.497193] ? lock_acquire+0xc1/0x2b0 [10705.497196] tcf_action_exec+0xac/0x230 [10705.497202] fl_classify+0x20e/0x260 [cls_flower] [10705.497210] ? __bfs+0xf3/0x240 [10705.497213] ? check_irq_usage+0x19c/0xc40 [10705.497215] ? __lock_acquire+0xa6e/0x1750 [10705.497218] ? lock_acquire+0x179/0x2b0 [10705.497220] ? check_path.constprop.0+0x24/0x50 [10705.497222] ? check_noncircular+0x81/0x150 [10705.497224] ? lock_acquire+0x179/0x2b0 [10705.497226] ? is_bpf_text_address+0x71/0x100 [10705.497230] ? lock_release+0x1d3/0x270 [10705.497233] ? __lock_acquire+0xa6e/0x1750 [10705.497236] tcf_classify+0x17a/0x540 [10705.497241] tc_run+0x9f/0x150 [10705.497246] __dev_queue_xmit+0x311/0x1180 [10705.497248] ? mark_held_locks+0x49/0x80 [10705.497250] ? eth_header+0x2a/0xc0 [10705.497254] ip_finish_output2+0x22d/0xa20 [10705.497258] ? __ip_queue_xmit+0x1d5/0x6f0 [10705.497260] __ip_queue_xmit+0x1d5/0x6f0 [10705.497263] __tcp_transmit_skb+0xb65/0xd30 [10705.497268] ? __alloc_skb+0xdc/0x1a0 [10705.497272] ? __pfx_tcp_delack_timer+0x10/0x10 [10705.497274] tcp_delack_timer_handler+0xa3/0xf0 [10705.497276] tcp_delack_timer+0x5a/0x140 [10705.497278] ? __pfx_tcp_delack_timer+0x10/0x10 [10705.497279] call_timer_fn+0xa5/0x220 [10705.497283] __run_timers.part.0+0x1f8/0x2c0 [10705.497286] ? kvm_clock_get_cycles+0x18/0x30 [10705.497290] run_timer_softirq+0x2a/0x50 [10705.497291] __do_softirq+0xc8/0x37a [10705.497294] irq_exit_rcu+0x7f/0xa0 [10705.497297] sysvec_apic_timer_interrupt+0x41/0x90 [10705.497300] asm_sysvec_apic_timer_interrupt+0x1a/0x20 [10705.497302] RIP: 0033:0x7fd2c2613700 [10705.497305] Code: ff ff 64 89 02 eb bd 0f 1f 00 f3 0f 1e fa 41 89 ca 64 8b 04 25 18 00 00 00 85 c0 75 1d 45 31 c9 45 31 c0 b8 2c 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 68 c3 0f 1f 80 00 00 00 00 41 54 48 83 ec 20 [10705.497306] RSP: 002b:00007ffe6ba49238 EFLAGS: 00000246 [10705.497307] RAX: 0000000000002000 RBX: 0000000000002000 RCX: 00007fd2c2613700 [10705.497308] RDX: 0000000000002000 RSI: 0000559104c92780 RDI: 0000000000000005 [10705.497309] RBP: 0000559104b589e0 R08: 0000000000000000 R09: 0000000000000000 [10705.497310] R10: 0000000000000000 R11: 0000000000000246 R12: 0000559104bbccf0 [10705.497311] R13: 0000559104bbcb10 R14: 0000559104bbcb10 R15: 0000559104b589e0 [10705.497314] </TASK> [10705.498050] CPU: 2 PID: 5626 Comm: ncat Not tainted 6.8.0-rc2-virtme #1 [10705.520025] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [10705.520616] RIP: 0010:queued_spin_lock_slowpath+0x1c/0x310 [10705.520900] Code: 90 90 90 90 90 90 90 90 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 41 55 41 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 <85> c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d c3 cc cc [10705.521862] RSP: 0018:ffffbef2401077a8 EFLAGS: 00000202 [10705.522137] RAX: 0000000000000001 RBX: ffff9a9dc2ac0098 RCX: 0000000000000001 [10705.522513] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9a9dc2ac0098 [10705.522873] RBP: ffff9a9dc2ac0098 R08: 0000000000000008 R09: ffffffff96d9d1a8 [10705.523255] R10: 0000000000000000 R11: 0000000000000001 R12: 0000000000000000 [10705.523626] R13: ffff9a9dc2ac0000 R14: ffff9a9dc2ac0098 R15: ffff9a9dc23ea94c [10705.523999] FS: 00007fa251394740(0000) GS:ffff9a9dfed00000(0000) knlGS:0000000000000000 [10705.524418] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [10705.524720] CR2: 00005653cd1c7340 CR3: 000000000263c001 CR4: 0000000000770ef0 [10705.525092] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [10705.525460] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [10705.525834] PKRU: 55555554 [10705.525981] Call Trace: [10705.526122] <IRQ> [10705.526236] ? rcu_dump_cpu_stacks+0xfa/0x180 [10705.526474] ? rcu_sched_clock_irq+0x7bb/0x13d0 [10705.526719] ? lock_is_held_type+0x9f/0x120 [10705.526942] ? lock_is_held_type+0x9f/0x120 [10705.527168] ? update_process_times+0x5f/0x90 [10705.527400] ? __pfx_tick_nohz_highres_handler+0x10/0x10 [10705.527682] ? tick_sched_handle+0x33/0x50 [10705.527905] ? tick_nohz_hi5828] </TASK> exit