====================================== | [955.640385][T11657] #2: ffff88800abc3a50 (&devlink->lock_key){+.+.}-{4:4}, at: devlink_get_from_attrs_lock (net/devlink/devl_internal.h:101 net/devlink/netlink.c:199) | [955.640762][T11657] #3: ffffffffb0fe1208 (rtnl_mutex){+.+.}-{4:4}, at: nsim_destroy (drivers/net/netdevsim/netdev.c:817) netdevsim | [ 955.641127][T11657] | [ 955.641127][T11657] stack backtrace: [ 955.641645][T11657] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 955.641912][T11657] Call Trace: [ 955.642057][T11657] [955.642157][T11657] dump_stack_lvl (lib/dump_stack.c:123) [955.642358][T11657] print_circular_bug (kernel/locking/lockdep.c:2077) [955.642550][T11657] check_noncircular (kernel/locking/lockdep.c:2206) [955.642738][T11657] ? __pfx_check_noncircular (kernel/locking/lockdep.c:2182) [955.642930][T11657] ? __kernel_text_address (kernel/extable.c:79) [955.643120][T11657] ? unwind_get_return_address (arch/x86/kernel/unwind_orc.c:369 arch/x86/kernel/unwind_orc.c:364) [955.643309][T11657] ? __pfx_stack_trace_consume_entry (kernel/stacktrace.c:83) [955.643544][T11657] ? arch_stack_walk (arch/x86/kernel/stacktrace.c:26) [955.643817][T11657] check_prev_add (kernel/locking/lockdep.c:3162) [955.644115][T11657] ? __pfx_stack_trace_save (kernel/stacktrace.c:114) [955.644366][T11657] validate_chain (kernel/locking/lockdep.c:3281 kernel/locking/lockdep.c:3904) [955.644667][T11657] ? __pfx_validate_chain (kernel/locking/lockdep.c:3860) [955.644966][T11657] ? mark_lock (kernel/locking/lockdep.c:186 kernel/locking/lockdep.c:4755) [955.645254][T11657] __lock_acquire (kernel/locking/lockdep.c:5226) [955.645524][T11657] ? kasan_addr_to_slab (./arch/x86/include/asm/jump_label.h:36 ./include/linux/page-flags.h:205 ./include/linux/page-flags.h:246 ./include/linux/mm.h:1296 mm/kasan/../slab.h:211 mm/kasan/common.c:38) [955.645822][T11657] lock_acquire.part.0 (kernel/locking/lockdep.c:467 kernel/locking/lockdep.c:5851) [955.646106][T11657] ? __kernfs_remove (fs/kernfs/dir.c:1492) [955.646411][T11657] ? __pfx___lock_release (kernel/locking/lockdep.c:5501) [955.646722][T11657] ? __pfx_lock_acquire.part.0 (kernel/locking/lockdep.c:5814) [955.647004][T11657] ? trace_lock_acquire (./include/trace/events/lock.h:24 (discriminator 37)) [955.647280][T11657] ? __virt_addr_valid (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:962 ./include/linux/mmzone.h:2058 arch/x86/mm/physaddr.c:65) [955.647566][T11657] ? lock_acquire (kernel/locking/lockdep.c:5822) [955.647841][T11657] ? __kernfs_remove (fs/kernfs/dir.c:1492) [955.648137][T11657] kernfs_drain (./include/linux/instrumented.h:68 ./include/linux/atomic/atomic-instrumented.h:32 fs/kernfs/dir.c:501) [955.648422][T11657] ? __kernfs_remove (fs/kernfs/dir.c:1492) [955.648707][T11657] ? __pfx_kernfs_drain (fs/kernfs/dir.c:481) [955.648980][T11657] ? lock_acquire (kernel/locking/lockdep.c:5822) [955.649234][T11657] __kernfs_remove (fs/kernfs/dir.c:1492) [955.649512][T11657] kernfs_remove_by_name_ns (fs/kernfs/dir.c:1695) [955.649796][T11657] remove_files (fs/sysfs/group.c:27 (discriminator 3)) [955.650079][T11657] sysfs_remove_group (fs/sysfs/group.c:323) [955.650372][T11657] sysfs_remove_groups (fs/sysfs/group.c:345 fs/sysfs/group.c:338) [955.650649][T11657] device_remove_attrs (drivers/base/core.c:2937) [955.650888][T11657] ? __pfx_device_remove_attrs (drivers/base/core.c:2937) [955.651179][T11657] device_del (drivers/base/core.c:3854) [955.651394][T11657] ? __pfx_device_del (drivers/base/core.c:3809) [955.651684][T11657] unregister_netdevice_many_notify (net/core/dev.c:2527 (discriminator 3) net/core/dev.c:11568 (discriminator 3)) [955.652038][T11657] ? __mutex_trylock_common (./arch/x86/include/asm/atomic64_64.h:101 ./include/linux/atomic/atomic-arch-fallback.h:4296 ./include/linux/atomic/atomic-long.h:1482 ./include/linux/atomic/atomic-instrumented.h:4458 kernel/locking/mutex.c:104) [955.652326][T11657] ? __pfx___mutex_trylock_common (kernel/locking/mutex.c:79) [955.652671][T11657] ? __pfx_unregister_netdevice_many_notify (net/core/dev.c:11472) [955.653021][T11657] ? __mutex_lock (./arch/x86/include/asm/preempt.h:94 kernel/locking/mutex.c:595 kernel/locking/mutex.c:735) [955.653301][T11657] ? nsim_destroy (drivers/net/netdevsim/netdev.c:817) netdevsim [955.653599][T11657] ? __pfx___mutex_lock (kernel/locking/mutex.c:734) [955.653884][T11657] unregister_netdevice_queue (net/core/dev.c:11467) [955.654165][T11657] ? __pfx_do_raw_spin_lock (kernel/locking/spinlock_debug.c:114) [955.654454][T11657] ? __pfx_unregister_netdevice_queue (net/core/dev.c:11456) [955.654798][T11657] nsim_destroy (drivers/net/netdevsim/netdev.c:822 (discriminator 2)) netdevsim [955.655098][T11657] __nsim_dev_port_del (drivers/net/netdevsim/dev.c:426 drivers/net/netdevsim/dev.c:1429) netdevsim [955.655464][T11657] nsim_dev_reload_destroy (drivers/net/netdevsim/dev.c:1438 drivers/net/netdevsim/dev.c:1661) netdevsim [955.655829][T11657] nsim_dev_reload_down (drivers/net/netdevsim/dev.c:969) netdevsim [955.656193][T11657] devlink_reload (net/devlink/dev.c:461) [955.656488][T11657] ? __pfx_devlink_reload (net/devlink/dev.c:446) [955.656765][T11657] ? fdget (./include/linux/atomic/atomic-arch-fallback.h:479 ./include/linux/atomic/atomic-instrumented.h:50 fs/file.c:1145 fs/file.c:1159) [955.656976][T11657] devlink_nl_reload_doit (net/devlink/dev.c:584) [955.657251][T11657] ? __pfx_devlink_nl_reload_doit (net/devlink/dev.c:520) [955.657598][T11657] ? __nla_parse (lib/nlattr.c:732) [955.657876][T11657] genl_family_rcv_msg_doit (net/netlink/genetlink.c:1115) [955.658152][T11657] ? __pfx_genl_family_rcv_msg_doit (net/netlink/genetlink.c:1088) [955.658507][T11657] ? validate_chain (kernel/locking/lockdep.c:3797 kernel/locking/lockdep.c:3817 kernel/locking/lockdep.c:3872) [955.658783][T11657] ? rcu_read_lock_any_held (kernel/rcu/update.c:386 kernel/rcu/update.c:380) [955.659066][T11657] ? validate_chain (kernel/locking/lockdep.c:3797 kernel/locking/lockdep.c:3817 kernel/locking/lockdep.c:3872) [955.659353][T11657] genl_family_rcv_msg (net/netlink/genetlink.c:1195) [955.659632][T11657] ? __pfx_genl_family_rcv_msg (net/netlink/genetlink.c:1160) [955.659904][T11657] ? __pfx_devlink_nl_pre_doit_dev_lock (net/devlink/netlink.c:257) [955.660244][T11657] ? __pfx_devlink_nl_reload_doit (net/devlink/dev.c:520) [955.660595][T11657] ? __pfx_devlink_nl_post_doit_dev_lock (net/devlink/netlink.c:288) [955.660934][T11657] genl_rcv_msg (net/netlink/genetlink.c:65 net/netlink/genetlink.c:1211) [955.661220][T11657] netlink_rcv_skb (net/netlink/af_netlink.c:2542) [955.661498][T11657] ? __pfx_genl_rcv_msg (net/netlink/genetlink.c:1201) [955.661772][T11657] ? __pfx_netlink_rcv_skb (net/netlink/af_netlink.c:2519) [955.662054][T11657] ? genl_rcv (net/netlink/genetlink.c:1219) [955.662258][T11657] ? __pfx_down_read (kernel/locking/rwsem.c:1522) [955.662552][T11657] ? netlink_deliver_tap (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/netlink/af_netlink.c:340) [955.662827][T11657] genl_rcv (net/netlink/genetlink.c:1220) [955.663039][T11657] netlink_unicast (net/netlink/af_netlink.c:1321 net/netlink/af_netlink.c:1347) [955.663312][T11657] ? __pfx_netlink_unicast (net/netlink/af_netlink.c:1332) [955.663587][T11657] ? find_held_lock (kernel/locking/lockdep.c:5339) [955.663872][T11657] netlink_sendmsg (net/netlink/af_netlink.c:1891) [955.664157][T11657] ? __pfx_netlink_sendmsg (net/netlink/af_netlink.c:1810) [955.664444][T11657] ? lock_acquire (kernel/locking/lockdep.c:5822) [955.664718][T11657] ? __might_fault (mm/memory.c:6751 mm/memory.c:6744) [955.664998][T11657] __sys_sendto (net/socket.c:711 net/socket.c:726 net/socket.c:2208) [955.665276][T11657] ? __pfx___sys_sendto (net/socket.c:2175) [955.665555][T11657] ? __pfx___lock_release (kernel/locking/lockdep.c:5501) [955.665825][T11657] ? trace_lock_acquire (./include/trace/events/lock.h:24 (discriminator 37)) [955.666101][T11657] ? rseq_update_cpu_node_id (kernel/rseq.c:109 (discriminator 10)) [955.666387][T11657] ? __rseq_handle_notify_resume (kernel/rseq.c:333) [955.666723][T11657] ? __pfx___rseq_handle_notify_resume (kernel/rseq.c:316) [955.667063][T11657] ? __pfx___x64_sys_openat (fs/open.c:1428) [955.667335][T11657] __x64_sys_sendto (net/socket.c:2211) [955.667606][T11657] ? lockdep_hardirqs_on_prepare (kernel/locking/lockdep.c:4347 kernel/locking/lockdep.c:4406) [955.667938][T11657] do_syscall_64 (arch/x86/entry/common.c:52 arch/x86/entry/common.c:83) [955.668202][T11657] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:130) [ 955.668541][T11657] RIP: 0033:0x7fd2f13b1a4a [ 955.668818][T11657] Code: d8 64 89 02 48 c7 c0 ff ff ff ff eb b8 0f 1f 00 f3 0f 1e fa 41 89 ca 64 8b 04 25 18 00 00 00 85 c0 75 15 b8 2c 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 7e c3 0f 1f 44 00 00 41 54 48 83 ec 30 44 89 All code ======== 0: d8 64 89 02 fsubs 0x2(%rcx,%rcx,4) 4: 48 c7 c0 ff ff ff ff mov $0xffffffffffffffff,%rax b: eb b8 jmp 0xffffffffffffffc5 d: 0f 1f 00 nopl (%rax) 10: f3 0f 1e fa endbr64 14: 41 89 ca mov %ecx,%r10d 17: 64 8b 04 25 18 00 00 mov %fs:0x18,%eax 1e: 00 1f: 85 c0 test %eax,%eax 21: 75 15 jne 0x38 23: b8 2c 00 00 00 mov $0x2c,%eax 28: 0f 05 syscall 2a:* 48 3d 00 f0 ff ff cmp $0xfffffffffffff000,%rax <-- trapping instruction 30: 77 7e ja 0xb0 32: c3 ret 33: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 38: 41 54 push %r12 3a: 48 83 ec 30 sub $0x30,%rsp 3e: 44 rex.R 3f: 89 .byte 0x89 Code starting with the faulting instruction =========================================== 0: 48 3d 00 f0 ff ff cmp $0xfffffffffffff000,%rax 6: 77 7e ja 0x86 8: c3 ret 9: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) e: 41 54 push %r12 10: 48 83 ec 30 sub $0x30,%rsp 14: 44 rex.R 15: 89 .byte 0x89 [ 955.669764][T11657] RSP: 002b:00007ffefa4979b8 EFLAGS: 00000246 ORIG_RAX: 000000000000002c [ 955.670164][T11657] RAX: ffffffffffffffda RBX: 00000000331cfb00 RCX: 00007fd2f13b1a4a [ 955.670577][T11657] RDX: 0000000000000040 RSI: 00000000331cfb00 RDI: 0000000000000005 [ 955.670985][T11657] RBP: 00000000331cf910 R08: 00007fd2f1555200 R09: 000000000000000c [ 955.671351][T11657] R10: 0000000000000000 R11: 0000000000000246 R12: 000000000040f200 [ 955.671622][T11657] R13: 00000000331cf910 R14: 0000000000000000 R15: 0000000000000001 | [ 955.670577][T11657] RDX: 0000000000000040 RSI: 00000000331cfb00 RDI: 0000000000000005 | [ 955.670985][T11657] RBP: 00000000331cf910 R08: 00007fd2f1555200 R09: 000000000000000c | [ 955.671351][T11657] R10: 0000000000000000 R11: 0000000000000246 R12: 000000000040f200 | [ 955.671622][T11657] R13: 00000000331cf910 R14: 0000000000000000 R15: 0000000000000001 | [ 955.671898][T11657] [ 1107.427499][ T43] INFO: task devlink:11657 blocked for more than 122 seconds. [ 1107.427834][ T43] Not tainted 6.13.0-rc2-virtme #1 [ 1107.428004][ T43] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 1107.428259][ T43] task:devlink state:D stack:23480 pid:11657 tgid:11657 ppid:11614 flags:0x00004002 [ 1107.428590][ T43] Call Trace: [ 1107.428698][ T43] [ 1107.428772][ T43] __schedule (kernel/sched/core.c:5369 kernel/sched/core.c:6756) [ 1107.428930][ T43] ? __pfx___schedule (kernel/sched/core.c:6637) [ 1107.429112][ T43] ? __pfx_do_raw_spin_lock (kernel/locking/spinlock_debug.c:114) [ 1107.429251][ T43] ? trace_lock_release (./include/trace/events/lock.h:69 (discriminator 37)) [ 1107.429388][ T43] schedule (./arch/x86/include/asm/preempt.h:84 kernel/sched/core.c:6834 kernel/sched/core.c:6848) [ 1107.429501][ T43] kernfs_drain (fs/kernfs/dir.c:505 (discriminator 9)) [ 1107.429645][ T43] ? __pfx_kernfs_drain (fs/kernfs/dir.c:481) [ 1107.429778][ T43] ? __pfx_autoremove_wake_function (kernel/sched/wait.c:383) [ 1107.429949][ T43] __kernfs_remove (fs/kernfs/dir.c:1492) [ 1107.430109][ T43] kernfs_remove_by_name_ns (fs/kernfs/dir.c:1695) [ 1107.430248][ T43] remove_files (fs/sysfs/group.c:27 (discriminator 3)) [ 1107.430385][ T43] sysfs_remove_group (fs/sysfs/group.c:323) [ 1107.430528][ T43] sysfs_remove_groups (fs/sysfs/group.c:345 fs/sysfs/group.c:338) [ 1107.430668][ T43] device_remove_attrs (drivers/base/core.c:2937) [ 1107.430805][ T43] ? __pfx_device_remove_attrs (drivers/base/core.c:2937) [ 1107.430941][ T43] device_del (drivers/base/core.c:3854) [ 1107.431052][ T43] ? __pfx_device_del (drivers/base/core.c:3809) [ 1107.431184][ T43] unregister_netdevice_many_notify (net/core/dev.c:2527 (discriminator 3) net/core/dev.c:11568 (discriminator 3)) [ 1107.431353][ T43] ? __mutex_trylock_common (./arch/x86/include/asm/atomic64_64.h:101 ./include/linux/atomic/atomic-arch-fallback.h:4296 ./include/linux/atomic/atomic-long.h:1482 ./include/linux/atomic/atomic-instrumented.h:4458 kernel/locking/mutex.c:104) [ 1107.431512][ T43] ? __pfx___mutex_trylock_common (kernel/locking/mutex.c:79) [ 1107.431680][ T43] ? __pfx_unregister_netdevice_many_notify (net/core/dev.c:11472) [ 1107.431846][ T43] ? __mutex_lock (./arch/x86/include/asm/preempt.h:94 kernel/locking/mutex.c:595 kernel/locking/mutex.c:735) [ 1107.431985][ T43] ? nsim_destroy (drivers/net/netdevsim/netdev.c:817) netdevsim [ 1107.432144][ T43] ? __pfx___mutex_lock (kernel/locking/mutex.c:734) [ 1107.432280][ T43] unregister_netdevice_queue (net/core/dev.c:11467) [ 1107.432414][ T43] ? __pfx_do_raw_spin_lock (kernel/locking/spinlock_debug.c:114) [ 1107.432557][ T43] ? __pfx_unregister_netdevice_queue (net/core/dev.c:11456) [ 1107.432728][ T43] nsim_destroy (drivers/net/netdevsim/netdev.c:822 (discriminator 2)) netdevsim [ 1107.432875][ T43] __nsim_dev_port_del (drivers/net/netdevsim/dev.c:426 drivers/net/netdevsim/dev.c:1429) netdevsim [ 1107.433059][ T43] nsim_dev_reload_destroy (drivers/net/netdevsim/dev.c:1438 drivers/net/netdevsim/dev.c:1661) netdevsim [ 1107.433238][ T43] nsim_dev_reload_down (drivers/net/netdevsim/dev.c:969) netdevsim [ 1107.433413][ T43] devlink_reload (net/devlink/dev.c:461) [ 1107.433567][ T43] ? __pfx_devlink_reload (net/devlink/dev.c:446) [ 1107.433702][ T43] ? fdget (./include/linux/atomic/atomic-arch-fallback.h:479 ./include/linux/atomic/atomic-instrumented.h:50 fs/file.c:1145 fs/file.c:1159) [ 1107.433807][ T43] devlink_nl_reload_doit (net/devlink/dev.c:584) [ 1107.433940][ T43] ? __pfx_devlink_nl_reload_doit (net/devlink/dev.c:520) [ 1107.434111][ T43] ? __nla_parse (lib/nlattr.c:732) [ 1107.434249][ T43] genl_family_rcv_msg_doit (net/netlink/genetlink.c:1115) [ 1107.434383][ T43] ? __pfx_genl_family_rcv_msg_doit (net/netlink/genetlink.c:1088) [ 1107.434559][ T43] ? validate_chain (kernel/locking/lockdep.c:3797 kernel/locking/lockdep.c:3817 kernel/locking/lockdep.c:3872) [ 1107.434700][ T43] ? rcu_read_lock_any_held (kernel/rcu/update.c:386 kernel/rcu/update.c:380) [ 1107.434834][ T43] ? validate_chain (kernel/locking/lockdep.c:3797 kernel/locking/lockdep.c:3817 kernel/locking/lockdep.c:3872) [ 1107.434967][ T43] genl_family_rcv_msg (net/netlink/genetlink.c:1195) [ 1107.435108][ T43] ? __pfx_genl_family_rcv_msg (net/netlink/genetlink.c:1160) [ 1107.435244][ T43] ? __pfx_devlink_nl_pre_doit_dev_lock (net/devlink/netlink.c:257) [ 1107.435411][ T43] ? __pfx_devlink_nl_reload_doit (net/devlink/dev.c:520) [ 1107.435597][ T43] ? __pfx_devlink_nl_post_doit_dev_lock (net/devlink/netlink.c:288) [ 1107.435766][ T43] genl_rcv_msg (net/netlink/genetlink.c:65 net/netlink/genetlink.c:1211) [ 1107.435899][ T43] netlink_rcv_skb (net/netlink/af_netlink.c:2542) [ 1107.436034][ T43] ? __pfx_genl_rcv_msg (net/netlink/genetlink.c:1201) [ 1107.436175][ T43] ? __pfx_netlink_rcv_skb (net/netlink/af_netlink.c:2519) [ 1107.436312][ T43] ? genl_rcv (net/netlink/genetlink.c:1219) [ 1107.436412][ T43] ? __pfx_down_read (kernel/locking/rwsem.c:1522) [ 1107.436557][ T43] ? netlink_deliver_tap (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/netlink/af_netlink.c:340) [ 1107.436693][ T43] genl_rcv (net/netlink/genetlink.c:1220) [ 1107.436793][ T43] netlink_unicast (net/netlink/af_netlink.c:1321 net/netlink/af_netlink.c:1347) [ 1107.436929][ T43] ? __pfx_netlink_unicast (net/netlink/af_netlink.c:1332) [ 1107.437068][ T43] ? find_held_lock (kernel/locking/lockdep.c:5339) [ 1107.437203][ T43] netlink_sendmsg (net/netlink/af_netlink.c:1891) [ 1107.437338][ T43] ? __pfx_netlink_sendmsg (net/netlink/af_netlink.c:1810) [ 1107.437478][ T43] ? lock_acquire (kernel/locking/lockdep.c:5822) [ 1107.437609][ T43] ? __might_fault (mm/memory.c:6751 mm/memory.c:6744) [ 1107.437747][ T43] __sys_sendto (net/socket.c:711 net/socket.c:726 net/socket.c:2208) [ 1107.437884][ T43] ? __pfx___sys_sendto (net/socket.c:2175) [ 1107.438017][ T43] ? __pfx___lock_release (kernel/locking/lockdep.c:5501) [ 1107.438155][ T43] ? trace_lock_acquire (./include/trace/events/lock.h:24 (discriminator 37)) [ 1107.438292][ T43] ? rseq_update_cpu_node_id (kernel/rseq.c:109 (discriminator 10)) [ 1107.438428][ T43] ? __rseq_handle_notify_resume (kernel/rseq.c:333) [ 1107.438603][ T43] ? __pfx___rseq_handle_notify_resume (kernel/rseq.c:316) [ 1107.438770][ T43] ? __pfx___x64_sys_openat (fs/open.c:1428) [ 1107.438906][ T43] __x64_sys_sendto (net/socket.c:2211) [ 1107.439044][ T43] ? lockdep_hardirqs_on_prepare (kernel/locking/lockdep.c:4347 kernel/locking/lockdep.c:4406) [ 1107.439209][ T43] do_syscall_64 (arch/x86/entry/common.c:52 arch/x86/entry/common.c:83) [ 1107.439342][ T43] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:130) [ 1107.439517][ T43] RIP: 0033:0x7fd2f13b1a4a [ 1107.439661][ T43] RSP: 002b:00007ffefa4979b8 EFLAGS: 00000246 ORIG_RAX: 000000000000002c [ 1107.439863][ T43] RAX: ffffffffffffffda RBX: 00000000331cfb00 RCX: 00007fd2f13b1a4a [ 1107.440068][ T43] RDX: 0000000000000040 RSI: 00000000331cfb00 RDI: 0000000000000005 [ 1107.440318][ T43] RBP: 00000000331cf910 R08: 00007fd2f1555200 R09: 000000000000000c [ 1107.440530][ T43] R10: 0000000000000000 R11: 0000000000000246 R12: 000000000040f200 Finger prints: __schedule:schedule:kernfs_drain:__kernfs_remove:kernfs_remove_by_name_ns print_circular_bug:check_noncircular:check_prev_add:validate_chain:__lock_acquire