[ 54.844615][ T478] ip (478) used greatest stack depth: 23712 bytes left [ 59.552279][ T73] [ 59.552385][ T73] ===================================================== [ 59.552546][ T73] WARNING: SOFTIRQ-safe -> SOFTIRQ-unsafe lock order detected [ 59.552732][ T73] 6.9.0-rc1-virtme #1 Not tainted [ 59.552866][ T73] ----------------------------------------------------- [ 59.553024][ T73] kworker/u16:1/73 [HC0[0]:SC0[2]:HE1:SE0] is trying to acquire: [ 59.553214][ T73] ffff8880052d0968 (&p->alloc_lock){+.+.}-{2:2}, at: __get_task_comm+0x27/0x70 [ 59.553453][ T73] [ 59.553453][ T73] and this task is already holding: [ 59.553640][ T73] ffff88802f63f198 (&ul->lock#2){+.-.}-{2:2}, at: rt_flush_dev+0x13b/0x670 [ 59.553873][ T73] which would create a new lock dependency: [ 59.554029][ T73] (&ul->lock#2){+.-.}-{2:2} -> (&p->alloc_lock){+.+.}-{2:2} [ 59.554225][ T73] [ 59.554225][ T73] but this new dependency connects a SOFTIRQ-irq-safe lock: [ 59.554442][ T73] (&ul->lock#2){+.-.}-{2:2} [ 59.554447][ T73] [ 59.554447][ T73] ... which became SOFTIRQ-irq-safe at: [ 59.554755][ T73] __lock_acquire+0x797/0x1570 [ 59.554891][ T73] lock_acquire.part.0+0xeb/0x330 [ 59.555018][ T73] _raw_spin_lock_bh+0x38/0x50 [ 59.555148][ T73] rt_add_uncached_list+0x55/0x1b0 [ 59.555280][ T73] rt_cache_route+0xfd/0x150 [ 59.555408][ T73] ip_route_input_slow+0x1001/0x2230 [ 59.555534][ T73] ip_route_input_noref+0xe6/0x230 [ 59.555662][ T73] ip_rcv_finish_core.constprop.0+0x461/0x1300 [ 59.555819][ T73] ip_rcv+0x199/0x740 [ 59.555914][ T73] __netif_receive_skb_one_core+0x166/0x1b0 [ 59.556074][ T73] process_backlog+0x103/0x670 [ 59.556204][ T73] __napi_poll.constprop.0+0xa2/0x460 [ 59.556334][ T73] net_rx_action+0x49f/0xc00 [ 59.556461][ T73] __do_softirq+0x1f8/0x5df [ 59.556588][ T73] do_softirq+0x4d/0xa0 [ 59.556685][ T73] __local_bh_enable_ip+0xf6/0x120 [ 59.556811][ T73] __dev_queue_xmit+0x6bc/0x16e0 [ 59.556940][ T73] ip6_finish_output2+0x59b/0xf60 [ 59.557065][ T73] ip6_finish_output+0x553/0xdf0 [ 59.557189][ T73] ip6_output+0x1f3/0x770 [ 59.557318][ T73] NF_HOOK.constprop.0+0xe2/0x680 [ 59.557447][ T73] mld_sendpack+0x62d/0xbc0 [ 59.557573][ T73] mld_ifc_work+0x32/0x1f0 [ 59.557699][ T73] process_one_work+0xe2c/0x1730 [ 59.557828][ T73] worker_thread+0x587/0xd30 [ 59.557954][ T73] kthread+0x28a/0x350 [ 59.558050][ T73] ret_from_fork+0x31/0x70 [ 59.558178][ T73] ret_from_fork_asm+0x1a/0x30 [ 59.558306][ T73] [ 59.558306][ T73] to a SOFTIRQ-irq-unsafe lock: [ 59.558466][ T73] (&p->alloc_lock){+.+.}-{2:2} [ 59.558470][ T73] [ 59.558470][ T73] ... which became SOFTIRQ-irq-unsafe at: [ 59.558776][ T73] ... [ 59.558777][ T73] __lock_acquire+0x797/0x1570 [ 59.558974][ T73] lock_acquire.part.0+0xeb/0x330 [ 59.559100][ T73] _raw_spin_lock+0x30/0x40 [ 59.559227][ T73] set_mems_allowed+0x1d/0x210 [ 59.559353][ T73] kernel_init_freeable+0x72/0x310 [ 59.559486][ T73] kernel_init+0x20/0x200 [ 59.559616][ T73] ret_from_fork+0x31/0x70 [ 59.559744][ T73] ret_from_fork_asm+0x1a/0x30 [ 59.559871][ T73] [ 59.559871][ T73] other info that might help us debug this: [ 59.559871][ T73] [ 59.560115][ T73] Possible interrupt unsafe locking scenario: [ 59.560115][ T73] [ 59.560299][ T73] CPU0 CPU1 [ 59.560428][ T73] ---- ---- [ 59.560554][ T73] lock(&p->alloc_lock); [ 59.560652][ T73] local_irq_disable(); [ 59.560810][ T73] lock(&ul->lock#2); [ 59.560970][ T73] lock(&p->alloc_lock); [ 59.561129][ T73] [ 59.561226][ T73] lock(&ul->lock#2); [ 59.561325][ T73] [ 59.561325][ T73] *** DEADLOCK *** [ 59.561325][ T73] [ 59.561511][ T73] 5 locks held by kworker/u16:1/73: [ 59.561646][ T73] #0: ffff88800190a948 ((wq_completion)netns){+.+.}-{0:0}, at: process_one_work+0xda6/0x1730 [ 59.561921][ T73] #1: ffffc900004efda0 (net_cleanup_work){+.+.}-{0:0}, at: process_one_work+0xde2/0x1730 [ 59.562147][ T73] #2: ffffffff9af9bc50 (pernet_ops_rwsem){++++}-{3:3}, at: cleanup_net+0xbc/0xb60 [ 59.562375][ T73] #3: ffffffff9afb7208 (rtnl_mutex){+.+.}-{3:3}, at: default_device_exit_batch+0x81/0x2d0 [ 59.562631][ T73] #4: ffff88802f63f198 (&ul->lock#2){+.-.}-{2:2}, at: rt_flush_dev+0x13b/0x670 [ 59.562857][ T73] [ 59.562857][ T73] the dependencies between SOFTIRQ-irq-safe lock and the holding lock: [ 59.563102][ T73] -> (&ul->lock#2){+.-.}-{2:2} { [ 59.563239][ T73] HARDIRQ-ON-W at: [ 59.563340][ T73] __lock_acquire+0x797/0x1570 [ 59.563504][ T73] lock_acquire.part.0+0xeb/0x330 [ 59.563665][ T73] _raw_spin_lock_bh+0x38/0x50 [ 59.563826][ T73] rt_add_uncached_list+0x55/0x1b0 [ 59.563988][ T73] rt_cache_route+0xfd/0x150 [ 59.564149][ T73] rt_set_nexthop.constprop.0+0x283/0xe90 [ 59.564341][ T73] __mkroute_output+0x4fa/0x1210 [ 59.564504][ T73] ip_route_output_key_hash+0x15d/0x2b0 [ 59.564697][ T73] ip_route_output_flow+0x23/0x140 [ 59.564859][ T73] udp_tunnel_dst_lookup+0x236/0x3b0 [ 59.565018][ T73] vxlan_xmit_one+0x11c9/0x4570 [vxlan] [ 59.565226][ T73] vxlan_xmit+0x9b6/0xf40 [vxlan] [ 59.565399][ T73] dev_hard_start_xmit+0x10e/0x360 [ 59.565560][ T73] __dev_queue_xmit+0xe79/0x16e0 [ 59.565719][ T73] ip6_finish_output2+0x59b/0xf60 [ 59.565878][ T73] ip6_finish_output+0x553/0xdf0 [ 59.566037][ T73] ip6_output+0x1f3/0x770 [ 59.566195][ T73] NF_HOOK.constprop.0+0xe2/0x680 [ 59.566359][ T73] mld_sendpack+0x62d/0xbc0 [ 59.566520][ T73] mld_ifc_work+0x32/0x1f0 [ 59.566681][ T73] process_one_work+0xe2c/0x1730 [ 59.566841][ T73] worker_thread+0x587/0xd30 [ 59.567001][ T73] kthread+0x28a/0x350 [ 59.567130][ T73] ret_from_fork+0x31/0x70 [ 59.567292][ T73] ret_from_fork_asm+0x1a/0x30 [ 59.567456][ T73] IN-SOFTIRQ-W at: [ 59.567552][ T73] __lock_acquire+0x797/0x1570 [ 59.567714][ T73] lock_acquire.part.0+0xeb/0x330 [ 59.567873][ T73] _raw_spin_lock_bh+0x38/0x50 [ 59.568033][ T73] rt_add_uncached_list+0x55/0x1b0 [ 59.568193][ T73] rt_cache_route+0xfd/0x150 [ 59.568356][ T73] ip_route_input_slow+0x1001/0x2230 [ 59.568518][ T73] ip_route_input_noref+0xe6/0x230 [ 59.568679][ T73] ip_rcv_finish_core.constprop.0+0x461/0x1300 [ 59.568870][ T73] ip_rcv+0x199/0x740 [ 59.569000][ T73] __netif_receive_skb_one_core+0x166/0x1b0 [ 59.569193][ T73] process_backlog+0x103/0x670 [ 59.569358][ T73] __napi_poll.constprop.0+0xa2/0x460 [ 59.569518][ T73] net_rx_action+0x49f/0xc00 [ 59.569677][ T73] __do_softirq+0x1f8/0x5df [ 59.569837][ T73] do_softirq+0x4d/0xa0 [ 59.569997][ T73] __local_bh_enable_ip+0xf6/0x120 [ 59.570156][ T73] __dev_queue_xmit+0x6bc/0x16e0 [ 59.570316][ T73] ip6_finish_output2+0x59b/0xf60 [ 59.570478][ T73] ip6_finish_output+0x553/0xdf0 [ 59.570635][ T73] ip6_output+0x1f3/0x770 [ 59.570793][ T73] NF_HOOK.constprop.0+0xe2/0x680 [ 59.570952][ T73] mld_sendpack+0x62d/0xbc0 [ 59.571112][ T73] mld_ifc_work+0x32/0x1f0 [ 59.571272][ T73] process_one_work+0xe2c/0x1730 [ 59.571435][ T73] worker_thread+0x587/0xd30 [ 59.571593][ T73] kthread+0x28a/0x350 [ 59.571733][ T73] ret_from_fork+0x31/0x70 [ 59.571892][ T73] ret_from_fork_asm+0x1a/0x30 [ 59.572124][ T73] INITIAL USE at: [ 59.572219][ T73] __lock_acquire+0x797/0x1570 [ 59.572380][ T73] lock_acquire.part.0+0xeb/0x330 [ 59.572542][ T73] _raw_spin_lock_bh+0x38/0x50 [ 59.572775][ T73] rt_add_uncached_list+0x55/0x1b0 [ 59.572934][ T73] rt_cache_route+0xfd/0x150 [ 59.573091][ T73] rt_set_nexthop.constprop.0+0x283/0xe90 [ 59.573354][ T73] __mkroute_output+0x4fa/0x1210 [ 59.573515][ T73] ip_route_output_key_hash+0x15d/0x2b0 [ 59.573678][ T73] ip_route_output_flow+0x23/0x140 [ 59.573837][ T73] udp_tunnel_dst_lookup+0x236/0x3b0 [ 59.574067][ T73] vxlan_xmit_one+0x11c9/0x4570 [vxlan] [ 59.574238][ T73] vxlan_xmit+0x9b6/0xf40 [vxlan] [ 59.574407][ T73] dev_hard_start_xmit+0x10e/0x360 [ 59.574644][ T73] __dev_queue_xmit+0xe79/0x16e0 [ 59.574802][ T73] ip6_finish_output2+0x59b/0xf60 [ 59.574958][ T73] ip6_finish_output+0x553/0xdf0 [ 59.575263][ T73] ip6_output+0x1f3/0x770 [ 59.575422][ T73] NF_HOOK.constprop.0+0xe2/0x680 [ 59.575580][ T73] mld_sendpack+0x62d/0xbc0 [ 59.575741][ T73] mld_ifc_work+0x32/0x1f0 [ 59.575899][ T73] process_one_work+0xe2c/0x1730 [ 59.576058][ T73] worker_thread+0x587/0xd30 [ 59.576216][ T73] kthread+0x28a/0x350 [ 59.576416][ T73] ret_from_fork+0x31/0x70 [ 59.576575][ T73] ret_from_fork_asm+0x1a/0x30 [ 59.576733][ T73] } [ 59.576797][ T73] ... key at: [] __key.7+0x0/0x40 [ 59.577036][ T73] [ 59.577036][ T73] the dependencies between the lock to be acquired [ 59.577037][ T73] and SOFTIRQ-irq-unsafe lock: [ 59.577372][ T73] -> (&p->alloc_lock){+.+.}-{2:2} { [ 59.577505][ T73] HARDIRQ-ON-W at: [ 59.577673][ T73] __lock_acquire+0x797/0x1570 [ 59.577833][ T73] lock_acquire.part.0+0xeb/0x330 [ 59.577998][ T73] _raw_spin_lock+0x30/0x40 [ 59.578157][ T73] set_mems_allowed+0x1d/0x210 [ 59.578390][ T73] kernel_init_freeable+0x72/0x310 [ 59.578559][ T73] kernel_init+0x20/0x200 [ 59.578718][ T73] ret_from_fork+0x31/0x70 [ 59.578950][ T73] ret_from_fork_asm+0x1a/0x30 [ 59.579108][ T73] SOFTIRQ-ON-W at: [ 59.579209][ T73] __lock_acquire+0x797/0x1570 [ 59.579369][ T73] lock_acquire.part.0+0xeb/0x330 [ 59.579602][ T73] _raw_spin_lock+0x30/0x40 [ 59.579762][ T73] set_mems_allowed+0x1d/0x210 [ 59.579920][ T73] kernel_init_freeable+0x72/0x310 [ 59.580153][ T73] kernel_init+0x20/0x200 [ 59.580310][ T73] ret_from_fork+0x31/0x70 [ 59.580477][ T73] ret_from_fork_asm+0x1a/0x30 [ 59.580636][ T73] INITIAL USE at: [ 59.580805][ T73] __lock_acquire+0x797/0x1570 [ 59.580965][ T73] lock_acquire.part.0+0xeb/0x330 [ 59.581127][ T73] _raw_spin_lock+0x30/0x40 [ 59.581361][ T73] set_mems_allowed+0x1d/0x210 [ 59.581521][ T73] kernel_init_freeable+0x72/0x310 [ 59.581687][ T73] kernel_init+0x20/0x200 [ 59.581857][ T73] ret_from_fork+0x31/0x70 [ 59.582090][ T73] ret_from_fork_asm+0x1a/0x30 [ 59.582253][ T73] } [ 59.582320][ T73] ... key at: [] __key.388+0x0/0x40 [ 59.582512][ T73] ... acquired at: [ 59.582677][ T73] check_prev_add+0x194/0xce0 [ 59.582813][ T73] validate_chain+0x71d/0x9b0 [ 59.582942][ T73] __lock_acquire+0xaf0/0x1570 [ 59.583071][ T73] lock_acquire.part.0+0xeb/0x330 [ 59.583270][ T73] _raw_spin_lock+0x30/0x40 [ 59.583398][ T73] __get_task_comm+0x27/0x70 [ 59.583526][ T73] ref_tracker_alloc+0x2ee/0x490 [ 59.583656][ T73] rt_flush_dev+0x371/0x670 [ 59.583856][ T73] fib_netdev_event+0x248/0x5a0 [ 59.583986][ T73] notifier_call_chain+0xcd/0x150 [ 59.584119][ T73] unregister_netdevice_many_notify+0x548/0x1190 [ 59.584278][ T73] default_device_exit_batch+0x234/0x2d0 [ 59.584507][ T73] cleanup_net+0x4f3/0xb60 [ 59.584635][ T73] process_one_work+0xe2c/0x1730 [ 59.584762][ T73] worker_thread+0x587/0xd30 [ 59.584891][ T73] kthread+0x28a/0x350 [ 59.584988][ T73] ret_from_fork+0x31/0x70 [ 59.585266][ T73] ret_from_fork_asm+0x1a/0x30 [ 59.585393][ T73] [ 59.585459][ T73] [ 59.585459][ T73] stack backtrace: [ 59.585619][ T73] CPU: 2 PID: 73 Comm: kworker/u16:1 Not tainted 6.9.0-rc1-virtme #1 [ 59.585818][ T73] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 59.586098][ T73] Workqueue: netns cleanup_net [ 59.586229][ T73] Call Trace: [ 59.586404][ T73] [ 59.586472][ T73] dump_stack_lvl+0x82/0xd0 [ 59.586604][ T73] print_bad_irq_dependency+0x957/0xd40 [ 59.586733][ T73] ? __pfx_print_bad_irq_dependency+0x10/0x10 [ 59.586966][ T73] ? __pfx_usage_skip+0x10/0x10 [ 59.587092][ T73] ? __bfs+0x246/0x650 [ 59.587188][ T73] ? __pfx_usage_match+0x10/0x10 [ 59.587316][ T73] check_irq_usage+0x596/0x850 [ 59.587443][ T73] ? arch_stack_walk+0x8c/0xf0 [ 59.587648][ T73] ? __pfx_check_irq_usage+0x10/0x10 [ 59.587776][ T73] ? __bfs+0x246/0x650 [ 59.587871][ T73] ? __pfx_hlock_conflict+0x10/0x10 [ 59.587998][ T73] ? check_path.constprop.0+0x24/0x50 [ 59.588198][ T73] ? check_noncircular+0x14c/0x3e0 [ 59.588324][ T73] ? __pfx_check_noncircular+0x10/0x10 [ 59.588453][ T73] ? find_held_lock+0x2c/0x110 [ 59.588581][ T73] ? check_prev_add+0x194/0xce0 [ 59.588711][ T73] check_prev_add+0x194/0xce0 [ 59.588839][ T73] ? validate_chain+0x130/0x9b0 [ 59.588968][ T73] validate_chain+0x71d/0x9b0 [ 59.589096][ T73] ? mark_lock+0x38/0x3e0 [ 59.589193][ T73] ? __pfx_validate_chain+0x10/0x10 [ 59.589389][ T73] ? hlock_class+0x4e/0x130 [ 59.589515][ T73] ? mark_lock+0x38/0x3e0 [ 59.589613][ T73] __lock_acquire+0xaf0/0x1570 [ 59.589741][ T73] lock_acquire.part.0+0xeb/0x330 [ 59.589871][ T73] ? __get_task_comm+0x27/0x70 [ 59.590071][ T73] ? mark_lock+0x38/0x3e0 [ 59.590169][ T73] ? __pfx_lock_acquire.part.0+0x10/0x10 [ 59.590297][ T73] ? trace_lock_acquire+0x135/0x1c0 [ 59.590423][ T73] ? __get_task_comm+0x27/0x70 [ 59.590619][ T73] ? lock_acquire+0x32/0xc0 [ 59.590746][ T73] ? __get_task_comm+0x27/0x70 [ 59.590873][ T73] _raw_spin_lock+0x30/0x40 [ 59.591003][ T73] ? __get_task_comm+0x27/0x70 [ 59.591204][ T73] __get_task_comm+0x27/0x70 [ 59.591331][ T73] ref_tracker_alloc+0x2ee/0x490 [ 59.591457][ T73] ? lock_acquire.part.0+0xeb/0x330 [ 59.591583][ T73] ? __pfx_ref_tracker_alloc+0x10/0x10 [ 59.591717][ T73] ? rt_flush_dev+0x371/0x670 [ 59.591847][ T73] ? fib_netdev_event+0x248/0x5a0 [ 59.591974][ T73] ? notifier_call_chain+0xcd/0x150 [ 59.592100][ T73] ? unregister_netdevice_many_notify+0x548/0x1190 [ 59.592261][ T73] ? default_device_exit_batch+0x234/0x2d0 [ 59.592491][ T73] ? cleanup_net+0x4f3/0xb60 [ 59.592619][ T73] ? process_one_work+0xe2c/0x1730 [ 59.592746][ T73] ? worker_thread+0x587/0xd30 [ 59.592883][ T73] ? kthread+0x28a/0x350 [ 59.593045][ T73] ? ret_from_fork+0x31/0x70 [ 59.593169][ T73] ? ret_from_fork_asm+0x1a/0x30 [ 59.593293][ T73] ? do_raw_spin_lock+0x131/0x270 [ 59.593421][ T73] ? __pfx_do_raw_spin_lock+0x10/0x10 [ 59.593615][ T73] ? lock_acquire+0x32/0xc0 [ 59.593737][ T73] ? rt_flush_dev+0x13b/0x670 [ 59.593863][ T73] rt_flush_dev+0x371/0x670 [ 59.593987][ T73] ? __timer_delete_sync+0xd3/0x110 [ 59.594184][ T73] ? __timer_delete_sync+0xa7/0x110 [ 59.594309][ T73] fib_netdev_event+0x248/0x5a0 [ 59.594432][ T73] notifier_call_chain+0xcd/0x150 [ 59.594560][ T73] unregister_netdevice_many_notify+0x548/0x1190 [ 59.594716][ T73] ? mutex_is_locked+0x17/0x50 [ 59.594843][ T73] ? __pfx_unregister_netdevice_many_notify+0x10/0x10 [ 59.594997][ T73] ? __pfx_unregister_netdevice_queue+0x10/0x10 [ 59.595151][ T73] default_device_exit_batch+0x234/0x2d0 [ 59.595346][ T73] ? __pfx_default_device_exit_batch+0x10/0x10 [ 59.595500][ T73] ? ops_exit_list+0xb4/0x170 [ 59.595628][ T73] cleanup_net+0x4f3/0xb60 [ 59.595770][ T73] ? __pfx_lock_acquire.part.0+0x10/0x10 [ 59.595968][ T73] ? __pfx_cleanup_net+0x10/0x10 [ 59.596095][ T73] ? trace_lock_acquire+0x135/0x1c0 [ 59.596224][ T73] ? process_one_work+0xde2/0x1730 [ 59.596351][ T73] ? lock_acquire+0x32/0xc0 [ 59.596547][ T73] ? process_one_work+0xde2/0x1730 [ 59.596674][ T73] process_one_work+0xe2c/0x1730 [ 59.596805][ T73] ? __pfx___lock_release+0x10/0x10 [ 59.596942][ T73] ? __pfx_process_one_work+0x10/0x10 [ 59.597067][ T73] ? assign_work+0x16c/0x240 [ 59.597263][ T73] worker_thread+0x587/0xd30 [ 59.597386][ T73] ? lockdep_hardirqs_on_prepare.part.0+0x1af/0x370 [ 59.597542][ T73] ? __pfx_worker_thread+0x10/0x10 [ 59.597666][ T73] ? __pfx_worker_thread+0x10/0x10 [ 59.597789][ T73] kthread+0x28a/0x350 [ 59.597885][ T73] ? __pfx_kthread+0x10/0x10 [ 59.598008][ T73] ret_from_fork+0x31/0x70 [ 59.598131][ T73] ? __pfx_kthread+0x10/0x10 [ 59.598323][ T73] ret_from_fork_asm+0x1a/0x30 [ 59.598450][ T73]