[ 73.654844][ T578] GACT probability NOT on [ 74.975621][ C2] [ 74.975926][ C2] ======================================================== [ 74.976289][ C2] WARNING: possible irq lock inversion dependency detected [ 74.976633][ C2] 6.9.0-rc1-virtme #1 Not tainted [ 74.976861][ C2] -------------------------------------------------------- [ 74.977193][ C2] mausezahn/592 just changed the state of lock: [ 74.977461][ C2] ffff888003e69198 (k-slock-AF_INET){+.-.}-{2:2}, at: ipv4_sk_update_pmtu+0xb3/0xb60 [ 74.977871][ C2] but this lock took another, SOFTIRQ-unsafe lock in the past: [ 74.978206][ C2] (&p->alloc_lock){+.+.}-{2:2} [ 74.978214][ C2] [ 74.978214][ C2] [ 74.978214][ C2] and interrupts could create inverse lock ordering between them. [ 74.978214][ C2] [ 74.979023][ C2] [ 74.979023][ C2] other info that might help us debug this: [ 74.979346][ C2] Possible interrupt unsafe locking scenario: [ 74.979346][ C2] [ 74.979679][ C2] CPU0 CPU1 [ 74.979898][ C2] ---- ---- [ 74.980120][ C2] lock(&p->alloc_lock); [ 74.980300][ C2] local_irq_disable(); [ 74.980584][ C2] lock(k-slock-AF_INET); [ 74.980863][ C2] lock(&p->alloc_lock); [ 74.981141][ C2] [ 74.981315][ C2] lock(k-slock-AF_INET); [ 74.981555][ C2] [ 74.981555][ C2] *** DEADLOCK *** [ 74.981555][ C2] [ 74.981904][ C2] 2 locks held by mausezahn/592: [ 74.982137][ C2] #0: ffffffff90b522e0 (rcu_read_lock){....}-{1:2}, at: process_backlog+0x1f8/0x670 [ 74.982568][ C2] #1: ffffffff90b522e0 (rcu_read_lock){....}-{1:2}, at: ip_local_deliver_finish+0x209/0x4b0 [ 74.983022][ C2] [ 74.983022][ C2] the shortest dependencies between 2nd lock and 1st lock: [ 74.983399][ C2] -> (&p->alloc_lock){+.+.}-{2:2} { [ 74.983630][ C2] HARDIRQ-ON-W at: [ 74.983810][ C2] __lock_acquire+0x797/0x1570 [ 74.984108][ C2] lock_acquire.part.0+0xeb/0x330 [ 74.984390][ C2] _raw_spin_lock+0x30/0x40 [ 74.984671][ C2] set_mems_allowed+0x1d/0x210 [ 74.984956][ C2] kernel_init_freeable+0x72/0x310 [ 74.985236][ C2] kernel_init+0x20/0x200 [ 74.985518][ C2] ret_from_fork+0x31/0x70 [ 74.985815][ C2] ret_from_fork_asm+0x1a/0x30 [ 74.986096][ C2] SOFTIRQ-ON-W at: [ 74.986268][ C2] __lock_acquire+0x797/0x1570 [ 74.986570][ C2] lock_acquire.part.0+0xeb/0x330 [ 74.986871][ C2] _raw_spin_lock+0x30/0x40 [ 74.987155][ C2] set_mems_allowed+0x1d/0x210 [ 74.987443][ C2] kernel_init_freeable+0x72/0x310 [ 74.987717][ C2] kernel_init+0x20/0x200 [ 74.988004][ C2] ret_from_fork+0x31/0x70 [ 74.988294][ C2] ret_from_fork_asm+0x1a/0x30 [ 74.988578][ C2] INITIAL USE at: [ 74.988752][ C2] __lock_acquire+0x797/0x1570 [ 74.989037][ C2] lock_acquire.part.0+0xeb/0x330 [ 74.989332][ C2] _raw_spin_lock+0x30/0x40 [ 74.989624][ C2] set_mems_allowed+0x1d/0x210 [ 74.989918][ C2] kernel_init_freeable+0x72/0x310 [ 74.990205][ C2] kernel_init+0x20/0x200 [ 74.990510][ C2] ret_from_fork+0x31/0x70 [ 74.990802][ C2] ret_from_fork_asm+0x1a/0x30 [ 74.991090][ C2] } [ 74.991211][ C2] ... key at: [] __key.387+0x0/0x40 [ 74.991558][ C2] ... acquired at: [ 74.991727][ C2] __lock_acquire+0xaf0/0x1570 [ 74.991968][ C2] lock_acquire.part.0+0xeb/0x330 [ 74.992191][ C2] _raw_spin_lock+0x30/0x40 [ 74.992423][ C2] __get_task_comm+0x27/0x70 [ 74.992657][ C2] ref_tracker_alloc+0x2ee/0x490 [ 74.992901][ C2] dst_init+0x84/0x4a0 [ 74.993077][ C2] dst_alloc+0x97/0x150 [ 74.993244][ C2] rt_dst_alloc+0x31/0x3b0 [ 74.993481][ C2] __mkroute_output+0x451/0x1070 [ 74.993711][ C2] ip_route_output_key_hash+0x15d/0x2b0 [ 74.993944][ C2] icmp_route_lookup.constprop.0+0x321/0xb30 [ 74.994229][ C2] __icmp_send+0x8e8/0x1910 [ 74.994461][ C2] ip_fragment.constprop.0+0x183/0x240 [ 74.994695][ C2] ip_output+0x16b/0x4f0 [ 74.994927][ C2] iptunnel_xmit+0x578/0x930 [ 74.995153][ C2] vxlan_xmit_one+0x2a6d/0x4570 [vxlan] [ 74.995412][ C2] vxlan_xmit+0x855/0xf40 [vxlan] [ 74.995663][ C2] dev_hard_start_xmit+0x10e/0x360 [ 74.995897][ C2] __dev_queue_xmit+0xe69/0x16d0 [ 74.996129][ C2] packet_snd+0xd08/0x1840 [ 74.996359][ C2] __sys_sendto+0x37d/0x410 [ 74.996598][ C2] __x64_sys_sendto+0xe0/0x1c0 [ 74.996823][ C2] do_syscall_64+0xc6/0x1e0 [ 74.997050][ C2] entry_SYSCALL_64_after_hwframe+0x6d/0x75 [ 74.997330][ C2] [ 74.997445][ C2] -> (k-slock-AF_INET){+.-.}-{2:2} { [ 74.997691][ C2] HARDIRQ-ON-W at: [ 74.997864][ C2] __lock_acquire+0x797/0x1570 [ 74.998151][ C2] lock_acquire.part.0+0xeb/0x330 [ 74.998441][ C2] _raw_spin_trylock+0x4b/0x70 [ 74.998735][ C2] icmp_reply+0x2e6/0xa60 [ 74.999027][ C2] icmp_echo.part.0+0x182/0x220 [ 74.999319][ C2] icmp_echo+0xcc/0x190 [ 74.999608][ C2] icmp_rcv+0x791/0xec0 [ 74.999897][ C2] ip_protocol_deliver_rcu+0x2e5/0x360 [ 75.000187][ C2] ip_local_deliver_finish+0x2c2/0x4b0 [ 75.000512][ C2] ip_local_deliver+0x19d/0x480 [ 75.000805][ C2] ip_rcv+0x564/0x740 [ 75.001037][ C2] __netif_receive_skb_one_core+0x166/0x1b0 [ 75.001387][ C2] process_backlog+0x103/0x670 [ 75.001674][ C2] __napi_poll.constprop.0+0xa2/0x460 [ 75.001964][ C2] net_rx_action+0x49f/0xc00 [ 75.002260][ C2] __do_softirq+0x1f8/0x5df [ 75.002550][ C2] do_softirq+0x4d/0xa0 [ 75.002844][ C2] __local_bh_enable_ip+0xf6/0x120 [ 75.003136][ C2] raw_sendmsg+0xb33/0x1950 [ 75.003421][ C2] __sys_sendto+0x332/0x410 [ 75.003714][ C2] __x64_sys_sendto+0xe0/0x1c0 [ 75.004006][ C2] do_syscall_64+0xc6/0x1e0 [ 75.004299][ C2] entry_SYSCALL_64_after_hwframe+0x6d/0x75 [ 75.004641][ C2] IN-SOFTIRQ-W at: [ 75.004820][ C2] __lock_acquire+0x797/0x1570 [ 75.005104][ C2] lock_acquire.part.0+0xeb/0x330 [ 75.005393][ C2] _raw_spin_lock+0x30/0x40 [ 75.005697][ C2] ipv4_sk_update_pmtu+0xb3/0xb60 [ 75.006001][ C2] __udp4_lib_err+0x7f6/0xa50 [ 75.006293][ C2] icmp_unreach+0x2c1/0xba0 [ 75.006583][ C2] icmp_rcv+0x791/0xec0 [ 75.006873][ C2] ip_protocol_deliver_rcu+0x2e5/0x360 [ 75.007160][ C2] ip_local_deliver_finish+0x2c2/0x4b0 [ 75.007446][ C2] ip_local_deliver+0x19d/0x480 [ 75.007742][ C2] ip_rcv+0x564/0x740 [ 75.007974][ C2] __netif_receive_skb_one_core+0x166/0x1b0 [ 75.008327][ C2] process_backlog+0x103/0x670 [ 75.008611][ C2] __napi_poll.constprop.0+0xa2/0x460 [ 75.008902][ C2] net_rx_action+0x49f/0xc00 [ 75.009189][ C2] __do_softirq+0x1f8/0x5df [ 75.009475][ C2] do_softirq+0x4d/0xa0 [ 75.009791][ C2] __local_bh_enable_ip+0xf6/0x120 [ 75.010073][ C2] __dev_queue_xmit+0x6ac/0x16d0 [ 75.010375][ C2] packet_snd+0xd08/0x1840 [ 75.010674][ C2] __sys_sendto+0x37d/0x410 [ 75.010968][ C2] __x64_sys_sendto+0xe0/0x1c0 [ 75.011244][ C2] do_syscall_64+0xc6/0x1e0 [ 75.011532][ C2] entry_SYSCALL_64_after_hwframe+0x6d/0x75 [ 75.011871][ C2] INITIAL USE at: [ 75.012044][ C2] __lock_acquire+0x797/0x1570 [ 75.012325][ C2] lock_acquire.part.0+0xeb/0x330 [ 75.012609][ C2] _raw_spin_trylock+0x4b/0x70 [ 75.012891][ C2] icmp_reply+0x2e6/0xa60 [ 75.013179][ C2] icmp_echo.part.0+0x182/0x220 [ 75.013465][ C2] icmp_echo+0xcc/0x190 [ 75.013851][ C2] icmp_rcv+0x791/0xec0 [ 75.014075][ C2] ip_protocol_deliver_rcu+0x2e5/0x360 [ 75.014358][ C2] ip_local_deliver_finish+0x2c2/0x4b0 [ 75.014637][ C2] ip_local_deliver+0x19d/0x480 [ 75.015076][ C2] ip_rcv+0x564/0x740 [ 75.015311][ C2] __netif_receive_skb_one_core+0x166/0x1b0 [ 75.015670][ C2] process_backlog+0x103/0x670 [ 75.016125][ C2] __napi_poll.constprop.0+0xa2/0x460 [ 75.016407][ C2] net_rx_action+0x49f/0xc00 [ 75.016688][ C2] __do_softirq+0x1f8/0x5df [ 75.016967][ C2] do_softirq+0x4d/0xa0 [ 75.017344][ C2] __local_bh_enable_ip+0xf6/0x120 [ 75.017635][ C2] raw_sendmsg+0xb33/0x1950 [ 75.017919][ C2] __sys_sendto+0x332/0x410 [ 75.018352][ C2] __x64_sys_sendto+0xe0/0x1c0 [ 75.018639][ C2] do_syscall_64+0xc6/0x1e0 [ 75.018935][ C2] entry_SYSCALL_64_after_hwframe+0x6d/0x75 [ 75.019438][ C2] } [ 75.019557][ C2] ... key at: [] af_family_kern_slock_keys+0x20/0x300 [ 75.019977][ C2] ... acquired at: [ 75.020151][ C2] mark_lock+0x28d/0x3e0 [ 75.020404][ C2] mark_usage+0xd9/0x2a0 [ 75.020637][ C2] __lock_acquire+0x797/0x1570 [ 75.020865][ C2] lock_acquire.part.0+0xeb/0x330 [ 75.021098][ C2] _raw_spin_lock+0x30/0x40 [ 75.021332][ C2] ipv4_sk_update_pmtu+0xb3/0xb60 [ 75.021723][ C2] __udp4_lib_err+0x7f6/0xa50 [ 75.021972][ C2] icmp_unreach+0x2c1/0xba0 [ 75.022214][ C2] icmp_rcv+0x791/0xec0 [ 75.022384][ C2] ip_protocol_deliver_rcu+0x2e5/0x360 [ 75.022776][ C2] ip_local_deliver_finish+0x2c2/0x4b0 [ 75.023008][ C2] ip_local_deliver+0x19d/0x480 [ 75.023240][ C2] ip_rcv+0x564/0x740 [ 75.023424][ C2] __netif_receive_skb_one_core+0x166/0x1b0 [ 75.023712][ C2] process_backlog+0x103/0x670 [ 75.024108][ C2] __napi_poll.constprop.0+0xa2/0x460 [ 75.024339][ C2] net_rx_action+0x49f/0xc00 [ 75.024572][ C2] __do_softirq+0x1f8/0x5df [ 75.024803][ C2] do_softirq+0x4d/0xa0 [ 75.025291][ C2] __local_bh_enable_ip+0xf6/0x120 [ 75.025519][ C2] __dev_queue_xmit+0x6ac/0x16d0 [ 75.025747][ C2] packet_snd+0xd08/0x1840 [ 75.025980][ C2] __sys_sendto+0x37d/0x410 [ 75.026376][ C2] __x64_sys_sendto+0xe0/0x1c0 [ 75.026609][ C2] do_syscall_64+0xc6/0x1e0 [ 75.026836][ C2] entry_SYSCALL_64_after_hwframe+0x6d/0x75 [ 75.027130][ C2] [ 75.027241][ C2] [ 75.027241][ C2] stack backtrace: [ 75.027522][ C2] CPU: 2 PID: 592 Comm: mausezahn Not tainted 6.9.0-rc1-virtme #1 [ 75.027872][ C2] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 75.028386][ C2] Call Trace: [ 75.028570][ C2] [ 75.028694][ C2] dump_stack_lvl+0x82/0xd0 [ 75.028937][ C2] print_irq_inversion_bug.part.0+0x3d9/0x570 [ 75.029227][ C2] ? kernel_text_address+0xce/0xe0 [ 75.029620][ C2] ? __pfx_print_irq_inversion_bug.part.0+0x10/0x10 [ 75.029911][ C2] ? __pfx_usage_skip+0x10/0x10 [ 75.030133][ C2] ? __pfx_usage_match+0x10/0x10 [ 75.030376][ C2] ? arch_stack_walk+0xa2/0xf0 [ 75.030766][ C2] mark_lock_irq+0x4cd/0xa10 [ 75.031000][ C2] ? __pfx_mark_lock_irq+0x10/0x10 [ 75.031229][ C2] ? __pfx_stack_trace_save+0x10/0x10 [ 75.031472][ C2] ? save_trace+0x8f/0x5b0 [ 75.031856][ C2] mark_lock+0x28d/0x3e0 [ 75.032036][ C2] mark_usage+0xd9/0x2a0 [ 75.032210][ C2] __lock_acquire+0x797/0x1570 [ 75.032444][ C2] lock_acquire.part.0+0xeb/0x330 [ 75.032683][ C2] ? ipv4_sk_update_pmtu+0xb3/0xb60 [ 75.033072][ C2] ? __pfx_lock_acquire.part.0+0x10/0x10 [ 75.033300][ C2] ? trace_lock_acquire+0x135/0x1c0 [ 75.033536][ C2] ? ipv4_sk_update_pmtu+0xb3/0xb60 [ 75.033769][ C2] ? lock_acquire+0x32/0xc0 [ 75.034159][ C2] ? ipv4_sk_update_pmtu+0xb3/0xb60 [ 75.034390][ C2] _raw_spin_lock+0x30/0x40 [ 75.034625][ C2] ? ipv4_sk_update_pmtu+0xb3/0xb60 [ 75.034863][ C2] ipv4_sk_update_pmtu+0xb3/0xb60 [ 75.035258][ C2] ? vxlan_err_lookup+0x23f/0x420 [vxlan] [ 75.035514][ C2] ? __pfx_ipv4_sk_update_pmtu+0x10/0x10 [ 75.035746][ C2] ? __udp4_lib_err_encap+0x59c/0x930 [ 75.035983][ C2] __udp4_lib_err+0x7f6/0xa50 [ 75.036218][ C2] icmp_unreach+0x2c1/0xba0 [ 75.036613][ C2] icmp_rcv+0x791/0xec0 [ 75.036792][ C2] ip_protocol_deliver_rcu+0x2e5/0x360 [ 75.037031][ C2] ip_local_deliver_finish+0x2c2/0x4b0 [ 75.037263][ C2] ip_local_deliver+0x19d/0x480 [ 75.037490][ C2] ? __pfx_ip_local_deliver+0x10/0x10 [ 75.037724][ C2] ? ip_rcv_finish_core.constprop.0+0x1ad/0x1300 [ 75.038013][ C2] ip_rcv+0x564/0x740 [ 75.038191][ C2] ? __pfx_ip_rcv+0x10/0x10 [ 75.038596][ C2] ? lock_acquire.part.0+0xeb/0x330 [ 75.038847][ C2] ? process_backlog+0x1f8/0x670 [ 75.039078][ C2] ? __pfx_ip_rcv+0x10/0x10 [ 75.039305][ C2] __netif_receive_skb_one_core+0x166/0x1b0 [ 75.039745][ C2] ? __pfx___netif_receive_skb_one_core+0x10/0x10 [ 75.040031][ C2] ? process_backlog+0x1f8/0x670 [ 75.040256][ C2] ? lock_acquire+0x32/0xc0 [ 75.040496][ C2] ? process_backlog+0x1f8/0x670 [ 75.040877][ C2] process_backlog+0x103/0x670 [ 75.041109][ C2] ? find_held_lock+0x2c/0x110 [ 75.041338][ C2] __napi_poll.constprop.0+0xa2/0x460 [ 75.041567][ C2] net_rx_action+0x49f/0xc00 [ 75.041796][ C2] ? __pfx_net_rx_action+0x10/0x10 [ 75.042178][ C2] ? ktime_get+0xb3/0x190 [ 75.042352][ C2] ? hrtimer_interrupt+0x311/0x770 [ 75.042583][ C2] ? hrtimer_interrupt+0x31d/0x770 [ 75.042814][ C2] __do_softirq+0x1f8/0x5df [ 75.043371][ C2] ? __dev_queue_xmit+0x697/0x16d0 [ 75.043618][ C2] do_softirq+0x4d/0xa0 [ 75.043800][ C2] [ 75.043929][ C2] [ 75.044047][ C2] __local_bh_enable_ip+0xf6/0x120 [ 75.044277][ C2] ? __dev_queue_xmit+0x697/0x16d0 [ 75.044510][ C2] __dev_queue_xmit+0x6ac/0x16d0 [ 75.044738][ C2] ? packet_parse_headers+0x231/0x9d0 [ 75.044964][ C2] ? packet_parse_headers+0x231/0x9d0 [ 75.045195][ C2] ? __pfx_sock_alloc_send_pskb+0x10/0x10 [ 75.045574][ C2] ? __pfx___dev_queue_xmit+0x10/0x10 [ 75.045802][ C2] ? dev_get_by_index+0x177/0x270 [ 75.046028][ C2] packet_snd+0xd08/0x1840 [ 75.046265][ C2] ? __pfx___lock_release+0x10/0x10 [ 75.046639][ C2] ? __pfx_packet_snd+0x10/0x10 [ 75.046862][ C2] ? __might_fault+0xc3/0x170 [ 75.047091][ C2] ? trace_raw_output_mmap_lock_acquire_returned+0x60/0x1e0 [ 75.047431][ C2] ? __might_fault+0x11b/0x170 [ 75.047811][ C2] __sys_sendto+0x37d/0x410 [ 75.048035][ C2] ? __pfx___sys_sendto+0x10/0x10 [ 75.048268][ C2] ? sock_ioctl+0x3bc/0x590 [ 75.048509][ C2] ? __up_read+0x1a7/0x780 [ 75.048913][ C2] ? __pfx___up_read+0x10/0x10 [ 75.049152][ C2] ? do_user_addr_fault+0x4b8/0xbe0 [ 75.049379][ C2] __x64_sys_sendto+0xe0/0x1c0 [ 75.049603][ C2] ? lockdep_hardirqs_on_prepare.part.0+0x1af/0x370 [ 75.050045][ C2] do_syscall_64+0xc6/0x1e0 [ 75.050276][ C2] entry_SYSCALL_64_after_hwframe+0x6d/0x75 [ 75.050582][ C2] RIP: 0033:0x7f3517b4485a [ 75.050820][ C2] Code: d8 64 89 02 48 c7 c0 ff ff ff ff eb b8 0f 1f 00 f3 0f 1e fa 41 89 ca 64 8b 04 25 18 00 00 00 85 c0 75 15 b8 2c 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 7e c3 0f 1f 44 00 00 41 54 48 83 ec 30 44 89 [ 75.051780][ C2] RSP: 002b:00007ffc606db818 EFLAGS: 00000246 ORIG_RAX: 000000000000002c [ 75.052290][ C2] RAX: ffffffffffffffda RBX: 000000000ebaa9d0 RCX: 00007f3517b4485a [ 75.052629][ C2] RDX: 00000000000003c5 RSI: 000000000ebab022 RDI: 0000000000000005 [ 75.052969][ C2] RBP: 000000000ebab022 R08: 00007ffc606db820 R09: 0000000000000014 [ 75.053301][ C2] R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000000 [ 75.053640][ C2] R13: 00000000000003c5 R14: 00007ffc606db820 R15: 0000000000000000 [ 75.053998][ C2]