[ 38.538703][ T352] GACT probability NOT on [ 39.905555][ C1] [ 39.905707][ C1] ======================================================== [ 39.905967][ C1] WARNING: possible irq lock inversion dependency detected [ 39.906200][ C1] 6.9.0-rc1-virtme #1 Not tainted [ 39.906356][ C1] -------------------------------------------------------- [ 39.906590][ C1] mausezahn/366 just changed the state of lock: [ 39.906788][ C1] ffff888003e53858 (k-slock-AF_INET){+.-.}-{2:2}, at: ipv4_sk_update_pmtu+0xb3/0xb60 [ 39.907077][ C1] but this lock took another, SOFTIRQ-unsafe lock in the past: [ 39.907305][ C1] (&p->alloc_lock){+.+.}-{2:2} [ 39.907310][ C1] [ 39.907310][ C1] [ 39.907310][ C1] and interrupts could create inverse lock ordering between them. [ 39.907310][ C1] [ 39.907888][ C1] [ 39.907888][ C1] other info that might help us debug this: [ 39.908116][ C1] Possible interrupt unsafe locking scenario: [ 39.908116][ C1] [ 39.908346][ C1] CPU0 CPU1 [ 39.908501][ C1] ---- ---- [ 39.908650][ C1] lock(&p->alloc_lock); [ 39.908772][ C1] local_irq_disable(); [ 39.908972][ C1] lock(k-slock-AF_INET); [ 39.909167][ C1] lock(&p->alloc_lock); [ 39.909362][ C1] [ 39.909485][ C1] lock(k-slock-AF_INET); [ 39.909644][ C1] [ 39.909644][ C1] *** DEADLOCK *** [ 39.909644][ C1] [ 39.909871][ C1] 2 locks held by mausezahn/366: [ 39.910028][ C1] #0: ffffffff9c5522e0 (rcu_read_lock){....}-{1:2}, at: process_backlog+0x1f8/0x670 [ 39.910314][ C1] #1: ffffffff9c5522e0 (rcu_read_lock){....}-{1:2}, at: ip_local_deliver_finish+0x209/0x4b0 [ 39.910626][ C1] [ 39.910626][ C1] the shortest dependencies between 2nd lock and 1st lock: [ 39.910899][ C1] -> (&p->alloc_lock){+.+.}-{2:2} { [ 39.911062][ C1] HARDIRQ-ON-W at: [ 39.911184][ C1] __lock_acquire+0x797/0x1570 [ 39.911391][ C1] lock_acquire.part.0+0xeb/0x330 [ 39.911592][ C1] _raw_spin_lock+0x30/0x40 [ 39.911793][ C1] set_mems_allowed+0x1d/0x210 [ 39.911991][ C1] kernel_init_freeable+0x72/0x310 [ 39.912192][ C1] kernel_init+0x20/0x200 [ 39.912390][ C1] ret_from_fork+0x31/0x70 [ 39.912595][ C1] ret_from_fork_asm+0x1a/0x30 [ 39.912791][ C1] SOFTIRQ-ON-W at: [ 39.912909][ C1] __lock_acquire+0x797/0x1570 [ 39.913105][ C1] lock_acquire.part.0+0xeb/0x330 [ 39.913301][ C1] _raw_spin_lock+0x30/0x40 [ 39.913497][ C1] set_mems_allowed+0x1d/0x210 [ 39.913692][ C1] kernel_init_freeable+0x72/0x310 [ 39.913891][ C1] kernel_init+0x20/0x200 [ 39.914082][ C1] ret_from_fork+0x31/0x70 [ 39.914277][ C1] ret_from_fork_asm+0x1a/0x30 [ 39.914470][ C1] INITIAL USE at: [ 39.914586][ C1] __lock_acquire+0x797/0x1570 [ 39.914781][ C1] lock_acquire.part.0+0xeb/0x330 [ 39.914982][ C1] _raw_spin_lock+0x30/0x40 [ 39.915176][ C1] set_mems_allowed+0x1d/0x210 [ 39.915367][ C1] kernel_init_freeable+0x72/0x310 [ 39.915559][ C1] kernel_init+0x20/0x200 [ 39.915771][ C1] ret_from_fork+0x31/0x70 [ 39.915964][ C1] ret_from_fork_asm+0x1a/0x30 [ 39.916161][ C1] } [ 39.916244][ C1] ... key at: [] __key.387+0x0/0x40 [ 39.916484][ C1] ... acquired at: [ 39.916603][ C1] __lock_acquire+0xaf0/0x1570 [ 39.916761][ C1] lock_acquire.part.0+0xeb/0x330 [ 39.916918][ C1] _raw_spin_lock+0x30/0x40 [ 39.917080][ C1] __get_task_comm+0x27/0x70 [ 39.917237][ C1] ref_tracker_alloc+0x2ee/0x490 [ 39.917400][ C1] dst_init+0x84/0x4a0 [ 39.917521][ C1] dst_alloc+0x97/0x150 [ 39.917642][ C1] rt_dst_alloc+0x31/0x3b0 [ 39.917798][ C1] __mkroute_output+0x451/0x1070 [ 39.917953][ C1] ip_route_output_key_hash+0x15d/0x2b0 [ 39.918108][ C1] icmp_route_lookup.constprop.0+0x321/0xb30 [ 39.918305][ C1] __icmp_send+0x8e8/0x1910 [ 39.918459][ C1] ip_fragment.constprop.0+0x183/0x240 [ 39.918622][ C1] ip_output+0x16b/0x4f0 [ 39.918779][ C1] iptunnel_xmit+0x578/0x930 [ 39.918938][ C1] vxlan_xmit_one+0x2a6d/0x4570 [vxlan] [ 39.919123][ C1] vxlan_xmit+0x855/0xf40 [vxlan] [ 39.919292][ C1] dev_hard_start_xmit+0x10e/0x360 [ 39.919447][ C1] __dev_queue_xmit+0xe69/0x16d0 [ 39.919601][ C1] packet_snd+0xd08/0x1840 [ 39.919764][ C1] __sys_sendto+0x37d/0x410 [ 39.919926][ C1] __x64_sys_sendto+0xe0/0x1c0 [ 39.920086][ C1] do_syscall_64+0xc6/0x1e0 [ 39.920242][ C1] entry_SYSCALL_64_after_hwframe+0x6d/0x75 [ 39.920439][ C1] [ 39.920520][ C1] -> (k-slock-AF_INET){+.-.}-{2:2} { [ 39.920679][ C1] HARDIRQ-ON-W at: [ 39.920797][ C1] __lock_acquire+0x797/0x1570 [ 39.920994][ C1] lock_acquire.part.0+0xeb/0x330 [ 39.921196][ C1] _raw_spin_lock_bh+0x38/0x50 [ 39.921389][ C1] lock_sock_nested+0x59/0xe0 [ 39.921584][ C1] sock_bindtoindex+0x25/0x50 [ 39.921779][ C1] udp_sock_create4+0x104/0x3f0 [udp_tunnel] [ 39.922012][ C1] vxlan_create_sock+0xbc/0x1d0 [vxlan] [ 39.922254][ C1] vxlan_socket_create+0xfb/0x500 [vxlan] [ 39.922501][ C1] __vxlan_sock_add+0x388/0x8d0 [vxlan] [ 39.922752][ C1] vxlan_open+0x151/0x250 [vxlan] [ 39.922960][ C1] __dev_open+0x221/0x430 [ 39.923156][ C1] __dev_change_flags+0x469/0x6c0 [ 39.923348][ C1] dev_change_flags+0x80/0x160 [ 39.923541][ C1] do_setlink+0x7d6/0x21e0 [ 39.923737][ C1] __rtnl_newlink+0xaa7/0xd80 [ 39.923937][ C1] rtnl_newlink+0x63/0xa0 [ 39.924127][ C1] rtnetlink_rcv_msg+0x2fb/0xbf0 [ 39.924321][ C1] netlink_rcv_skb+0x130/0x360 [ 39.924520][ C1] netlink_unicast+0x449/0x710 [ 39.924715][ C1] netlink_sendmsg+0x723/0xbe0 [ 39.924909][ C1] ____sys_sendmsg+0x7b2/0xa10 [ 39.925108][ C1] ___sys_sendmsg+0xee/0x170 [ 39.925309][ C1] __sys_sendmsg+0xcd/0x170 [ 39.925503][ C1] do_syscall_64+0xc6/0x1e0 [ 39.925711][ C1] entry_SYSCALL_64_after_hwframe+0x6d/0x75 [ 39.925950][ C1] IN-SOFTIRQ-W at: [ 39.926068][ C1] __lock_acquire+0x797/0x1570 [ 39.926265][ C1] lock_acquire.part.0+0xeb/0x330 [ 39.926467][ C1] _raw_spin_lock+0x30/0x40 [ 39.926670][ C1] ipv4_sk_update_pmtu+0xb3/0xb60 [ 39.926865][ C1] __udp4_lib_err+0x7f6/0xa50 [ 39.927066][ C1] icmp_unreach+0x2c1/0xba0 [ 39.927270][ C1] icmp_rcv+0x791/0xec0 [ 39.927467][ C1] ip_protocol_deliver_rcu+0x2e5/0x360 [ 39.927664][ C1] ip_local_deliver_finish+0x2c2/0x4b0 [ 39.927870][ C1] ip_local_deliver+0x19d/0x480 [ 39.928069][ C1] ip_rcv+0x564/0x740 [ 39.928227][ C1] __netif_receive_skb_one_core+0x166/0x1b0 [ 39.928467][ C1] process_backlog+0x103/0x670 [ 39.928664][ C1] __napi_poll.constprop.0+0xa2/0x460 [ 39.928862][ C1] net_rx_action+0x49f/0xc00 [ 39.929063][ C1] __do_softirq+0x1f8/0x5df [ 39.929260][ C1] do_softirq+0x4d/0xa0 [ 39.929458][ C1] __local_bh_enable_ip+0xf6/0x120 [ 39.929652][ C1] __dev_queue_xmit+0x6ac/0x16d0 [ 39.929850][ C1] packet_snd+0xd08/0x1840 [ 39.930048][ C1] __sys_sendto+0x37d/0x410 [ 39.930248][ C1] __x64_sys_sendto+0xe0/0x1c0 [ 39.930446][ C1] do_syscall_64+0xc6/0x1e0 [ 39.930645][ C1] entry_SYSCALL_64_after_hwframe+0x6d/0x75 [ 39.930881][ C1] INITIAL USE at: [ 39.930999][ C1] __lock_acquire+0x797/0x1570 [ 39.931290][ C1] lock_acquire.part.0+0xeb/0x330 [ 39.931488][ C1] _raw_spin_lock_bh+0x38/0x50 [ 39.931686][ C1] lock_sock_nested+0x59/0xe0 [ 39.931879][ C1] sock_bindtoindex+0x25/0x50 [ 39.932164][ C1] udp_sock_create4+0x104/0x3f0 [udp_tunnel] [ 39.932403][ C1] vxlan_create_sock+0xbc/0x1d0 [vxlan] [ 39.932611][ C1] vxlan_socket_create+0xfb/0x500 [vxlan] [ 39.933042][ C1] __vxlan_sock_add+0x388/0x8d0 [vxlan] [ 39.933258][ C1] vxlan_open+0x151/0x250 [vxlan] [ 39.933464][ C1] __dev_open+0x221/0x430 [ 39.933752][ C1] __dev_change_flags+0x469/0x6c0 [ 39.933944][ C1] dev_change_flags+0x80/0x160 [ 39.934139][ C1] do_setlink+0x7d6/0x21e0 [ 39.934335][ C1] __rtnl_newlink+0xaa7/0xd80 [ 39.934621][ C1] rtnl_newlink+0x63/0xa0 [ 39.934826][ C1] rtnetlink_rcv_msg+0x2fb/0xbf0 [ 39.935024][ C1] netlink_rcv_skb+0x130/0x360 [ 39.935309][ C1] netlink_unicast+0x449/0x710 [ 39.935503][ C1] netlink_sendmsg+0x723/0xbe0 [ 39.935709][ C1] ____sys_sendmsg+0x7b2/0xa10 [ 39.935992][ C1] ___sys_sendmsg+0xee/0x170 [ 39.936190][ C1] __sys_sendmsg+0xcd/0x170 [ 39.936387][ C1] do_syscall_64+0xc6/0x1e0 [ 39.936586][ C1] entry_SYSCALL_64_after_hwframe+0x6d/0x75 [ 39.936910][ C1] } [ 39.936988][ C1] ... key at: [] af_family_kern_slock_keys+0x20/0x300 [ 39.937260][ C1] ... acquired at: [ 39.937379][ C1] mark_lock+0x28d/0x3e0 [ 39.937626][ C1] mark_usage+0xd9/0x2a0 [ 39.937790][ C1] __lock_acquire+0x797/0x1570 [ 39.937946][ C1] lock_acquire.part.0+0xeb/0x330 [ 39.938104][ C1] _raw_spin_lock+0x30/0x40 [ 39.938348][ C1] ipv4_sk_update_pmtu+0xb3/0xb60 [ 39.938505][ C1] __udp4_lib_err+0x7f6/0xa50 [ 39.938663][ C1] icmp_unreach+0x2c1/0xba0 [ 39.938819][ C1] icmp_rcv+0x791/0xec0 [ 39.939023][ C1] ip_protocol_deliver_rcu+0x2e5/0x360 [ 39.939185][ C1] ip_local_deliver_finish+0x2c2/0x4b0 [ 39.939340][ C1] ip_local_deliver+0x19d/0x480 [ 39.939496][ C1] ip_rcv+0x564/0x740 [ 39.939615][ C1] __netif_receive_skb_one_core+0x166/0x1b0 [ 39.939899][ C1] process_backlog+0x103/0x670 [ 39.940059][ C1] __napi_poll.constprop.0+0xa2/0x460 [ 39.940217][ C1] net_rx_action+0x49f/0xc00 [ 39.940376][ C1] __do_softirq+0x1f8/0x5df [ 39.940624][ C1] do_softirq+0x4d/0xa0 [ 39.940742][ C1] __local_bh_enable_ip+0xf6/0x120 [ 39.940898][ C1] __dev_queue_xmit+0x6ac/0x16d0 [ 39.941053][ C1] packet_snd+0xd08/0x1840 [ 39.941298][ C1] __sys_sendto+0x37d/0x410 [ 39.941457][ C1] __x64_sys_sendto+0xe0/0x1c0 [ 39.941612][ C1] do_syscall_64+0xc6/0x1e0 [ 39.941771][ C1] entry_SYSCALL_64_after_hwframe+0x6d/0x75 [ 39.942060][ C1] [ 39.942142][ C1] [ 39.942142][ C1] stack backtrace: [ 39.942339][ C1] CPU: 1 PID: 366 Comm: mausezahn Not tainted 6.9.0-rc1-virtme #1 [ 39.942577][ C1] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 39.943017][ C1] Call Trace: [ 39.943138][ C1] [ 39.943218][ C1] dump_stack_lvl+0x82/0xd0 [ 39.943380][ C1] print_irq_inversion_bug.part.0+0x3d9/0x570 [ 39.943669][ C1] ? kernel_text_address+0xce/0xe0 [ 39.943828][ C1] ? __pfx_print_irq_inversion_bug.part.0+0x10/0x10 [ 39.944025][ C1] ? __pfx_usage_skip+0x10/0x10 [ 39.944183][ C1] ? __pfx_usage_match+0x10/0x10 [ 39.944430][ C1] ? arch_stack_walk+0xa2/0xf0 [ 39.944593][ C1] mark_lock_irq+0x4cd/0xa10 [ 39.944752][ C1] ? __pfx_mark_lock_irq+0x10/0x10 [ 39.944911][ C1] ? __pfx_stack_trace_save+0x10/0x10 [ 39.945247][ C1] ? save_trace+0x8f/0x5b0 [ 39.945402][ C1] mark_lock+0x28d/0x3e0 [ 39.945520][ C1] ? __lock_unpin_lock+0x137/0x2b0 [ 39.945674][ C1] mark_usage+0xd9/0x2a0 [ 39.945812][ C1] __lock_acquire+0x797/0x1570 [ 39.946056][ C1] ? usage_skip+0x182/0x190 [ 39.946211][ C1] lock_acquire.part.0+0xeb/0x330 [ 39.946365][ C1] ? ipv4_sk_update_pmtu+0xb3/0xb60 [ 39.946521][ C1] ? __pfx_lock_acquire.part.0+0x10/0x10 [ 39.946763][ C1] ? trace_lock_acquire+0x135/0x1c0 [ 39.946917][ C1] ? ipv4_sk_update_pmtu+0xb3/0xb60 [ 39.947073][ C1] ? lock_acquire+0x32/0xc0 [ 39.947226][ C1] ? ipv4_sk_update_pmtu+0xb3/0xb60 [ 39.947468][ C1] _raw_spin_lock+0x30/0x40 [ 39.947621][ C1] ? ipv4_sk_update_pmtu+0xb3/0xb60 [ 39.947775][ C1] ipv4_sk_update_pmtu+0xb3/0xb60 [ 39.947934][ C1] ? vxlan_err_lookup+0x23f/0x420 [vxlan] [ 39.948105][ C1] ? __pfx_ipv4_sk_update_pmtu+0x10/0x10 [ 39.948345][ C1] ? __udp4_lib_err_encap+0x59c/0x930 [ 39.948503][ C1] __udp4_lib_err+0x7f6/0xa50 [ 39.948660][ C1] icmp_unreach+0x2c1/0xba0 [ 39.948817][ C1] icmp_rcv+0x791/0xec0 [ 39.949021][ C1] ip_protocol_deliver_rcu+0x2e5/0x360 [ 39.949182][ C1] ip_local_deliver_finish+0x2c2/0x4b0 [ 39.949341][ C1] ip_local_deliver+0x19d/0x480 [ 39.949499][ C1] ? __pfx_ip_local_deliver+0x10/0x10 [ 39.949745][ C1] ? ip_rcv_finish_core.constprop.0+0x1ad/0x1300 [ 39.949948][ C1] ip_rcv+0x564/0x740 [ 39.950067][ C1] ? __pfx_ip_rcv+0x10/0x10 [ 39.950226][ C1] ? lock_acquire.part.0+0xeb/0x330 [ 39.950384][ C1] ? process_backlog+0x1f8/0x670 [ 39.950627][ C1] ? __pfx_ip_rcv+0x10/0x10 [ 39.950788][ C1] __netif_receive_skb_one_core+0x166/0x1b0 [ 39.950986][ C1] ? __pfx___netif_receive_skb_one_core+0x10/0x10 [ 39.951270][ C1] ? process_backlog+0x1f8/0x670 [ 39.951427][ C1] ? lock_acquire+0x32/0xc0 [ 39.951584][ C1] ? process_backlog+0x1f8/0x670 [ 39.951744][ C1] process_backlog+0x103/0x670 [ 39.951903][ C1] ? find_held_lock+0x2c/0x110 [ 39.952148][ C1] __napi_poll.constprop.0+0xa2/0x460 [ 39.952311][ C1] net_rx_action+0x49f/0xc00 [ 39.952470][ C1] ? __pfx_net_rx_action+0x10/0x10 [ 39.952627][ C1] ? ktime_get+0xb3/0x190 [ 39.952834][ C1] ? hrtimer_interrupt+0x311/0x770 [ 39.952991][ C1] ? hrtimer_interrupt+0x31d/0x770 [ 39.953147][ C1] __do_softirq+0x1f8/0x5df [ 39.953306][ C1] ? __dev_queue_xmit+0x697/0x16d0 [ 39.953550][ C1] do_softirq+0x4d/0xa0 [ 39.953673][ C1] [ 39.953758][ C1] [ 39.953837][ C1] __local_bh_enable_ip+0xf6/0x120 [ 39.953990][ C1] ? __dev_queue_xmit+0x697/0x16d0 [ 39.954148][ C1] __dev_queue_xmit+0x6ac/0x16d0 [ 39.954392][ C1] ? packet_parse_headers+0x231/0x9d0 [ 39.954549][ C1] ? packet_parse_headers+0x231/0x9d0 [ 39.954708][ C1] ? __pfx_sock_alloc_send_pskb+0x10/0x10 [ 39.954870][ C1] ? __pfx___dev_queue_xmit+0x10/0x10 [ 39.955116][ C1] ? dev_get_by_index+0x177/0x270 [ 39.955274][ C1] packet_snd+0xd08/0x1840 [ 39.955433][ C1] ? __pfx___lock_release+0x10/0x10 [ 39.955589][ C1] ? __pfx_packet_snd+0x10/0x10 [ 39.955853][ C1] ? __might_fault+0xc3/0x170 [ 39.956011][ C1] ? trace_raw_output_mmap_lock_acquire_returned+0x60/0x1e0 [ 39.956246][ C1] ? __might_fault+0x11b/0x170 [ 39.956405][ C1] __sys_sendto+0x37d/0x410 [ 39.956652][ C1] ? __pfx___sys_sendto+0x10/0x10 [ 39.956812][ C1] ? sock_ioctl+0x3bc/0x590 [ 39.956969][ C1] ? __up_read+0x1a7/0x780 [ 39.957126][ C1] ? __pfx___up_read+0x10/0x10 [ 39.957455][ C1] ? do_user_addr_fault+0x4b8/0xbe0 [ 39.957614][ C1] __x64_sys_sendto+0xe0/0x1c0 [ 39.957770][ C1] ? lockdep_hardirqs_on_prepare.part.0+0x1af/0x370 [ 39.957964][ C1] do_syscall_64+0xc6/0x1e0 [ 39.958214][ C1] entry_SYSCALL_64_after_hwframe+0x6d/0x75 [ 39.958410][ C1] RIP: 0033:0x7f9ae930b85a [ 39.958576][ C1] Code: d8 64 89 02 48 c7 c0 ff ff ff ff eb b8 0f 1f 00 f3 0f 1e fa 41 89 ca 64 8b 04 25 18 00 00 00 85 c0 75 15 b8 2c 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 7e c3 0f 1f 44 00 00 41 54 48 83 ec 30 44 89 [ 39.959220][ C1] RSP: 002b:00007ffea67cf628 EFLAGS: 00000246 ORIG_RAX: 000000000000002c [ 39.959459][ C1] RAX: ffffffffffffffda RBX: 0000000017aa29d0 RCX: 00007f9ae930b85a [ 39.959784][ C1] RDX: 00000000000003c5 RSI: 0000000017aa3022 RDI: 0000000000000005 [ 39.960020][ C1] RBP: 0000000017aa3022 R08: 00007ffea67cf630 R09: 0000000000000014 [ 39.960252][ C1] R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000000 [ 39.960572][ C1] R13: 00000000000003c5 R14: 00007ffea67cf630 R15: 0000000000000000 [ 39.960816][ C1]