====================================== | [ 312.487838][ T228] #5: ffff888005e13a50 (&devlink->lock_key){+.+.}-{4:4}, at: nsim_drv_remove (drivers/net/netdevsim/dev.c:1676) netdevsim | [ 312.488210][ T228] #6: ffffffffb9de1248 (rtnl_mutex){+.+.}-{4:4}, at: nsim_destroy (drivers/net/netdevsim/netdev.c:817) netdevsim | [ 312.488694][ T228] | [ 312.488694][ T228] stack backtrace: [ 312.489259][ T228] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 312.489489][ T228] Call Trace: [ 312.489623][ T228] [ 312.489716][ T228] dump_stack_lvl (lib/dump_stack.c:123) [ 312.489940][ T228] print_circular_bug (kernel/locking/lockdep.c:2077) [ 312.490150][ T228] check_noncircular (kernel/locking/lockdep.c:2206) [ 312.490344][ T228] ? __pfx_check_noncircular (kernel/locking/lockdep.c:2182) [ 312.490526][ T228] ? __kernel_text_address (kernel/extable.c:79) [ 312.490709][ T228] ? unwind_get_return_address (arch/x86/kernel/unwind_orc.c:369 arch/x86/kernel/unwind_orc.c:364) [ 312.490910][ T228] ? __pfx_stack_trace_consume_entry (kernel/stacktrace.c:83) [ 312.491148][ T228] ? arch_stack_walk (arch/x86/kernel/stacktrace.c:26) [ 312.491351][ T228] ? alloc_chain_hlocks (kernel/locking/lockdep.c:3528) [ 312.491583][ T228] check_prev_add (kernel/locking/lockdep.c:3162) [ 312.491764][ T228] ? __pfx_stack_trace_save (kernel/stacktrace.c:114) [ 312.491944][ T228] validate_chain (kernel/locking/lockdep.c:3281 kernel/locking/lockdep.c:3904) [ 312.492120][ T228] ? __pfx_validate_chain (kernel/locking/lockdep.c:3860) [ 312.492322][ T228] ? mark_lock (kernel/locking/lockdep.c:186 kernel/locking/lockdep.c:4755) [ 312.492498][ T228] __lock_acquire (kernel/locking/lockdep.c:5226) [ 312.492670][ T228] ? kasan_addr_to_slab (./arch/x86/include/asm/jump_label.h:36 ./include/linux/page-flags.h:205 ./include/linux/page-flags.h:246 ./include/linux/mm.h:1296 mm/kasan/../slab.h:211 mm/kasan/common.c:38) [ 312.492858][ T228] lock_acquire.part.0 (kernel/locking/lockdep.c:467 kernel/locking/lockdep.c:5851) [ 312.493041][ T228] ? __kernfs_remove (fs/kernfs/dir.c:1492) [ 312.493232][ T228] ? __pfx___lock_release (kernel/locking/lockdep.c:5501) [ 312.493417][ T228] ? __pfx_lock_acquire.part.0 (kernel/locking/lockdep.c:5814) [ 312.493597][ T228] ? trace_lock_acquire (./include/trace/events/lock.h:24 (discriminator 37)) [ 312.493776][ T228] ? __virt_addr_valid (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:962 ./include/linux/mmzone.h:2058 arch/x86/mm/physaddr.c:65) [ 312.493955][ T228] ? lock_acquire (kernel/locking/lockdep.c:5822) [ 312.494165][ T228] ? __kernfs_remove (fs/kernfs/dir.c:1492) [ 312.494458][ T228] kernfs_drain (./include/linux/instrumented.h:68 ./include/linux/atomic/atomic-instrumented.h:32 fs/kernfs/dir.c:501) [ 312.494664][ T228] ? __kernfs_remove (fs/kernfs/dir.c:1492) [ 312.494849][ T228] ? __pfx_kernfs_drain (fs/kernfs/dir.c:481) [ 312.495036][ T228] ? lock_acquire (kernel/locking/lockdep.c:5822) [ 312.495238][ T228] __kernfs_remove (fs/kernfs/dir.c:1492) [ 312.495430][ T228] kernfs_remove_by_name_ns (fs/kernfs/dir.c:1695) [ 312.495621][ T228] remove_files (fs/sysfs/group.c:27 (discriminator 3)) [ 312.495805][ T228] sysfs_remove_group (fs/sysfs/group.c:323) [ 312.496006][ T228] sysfs_remove_groups (fs/sysfs/group.c:345 fs/sysfs/group.c:338) [ 312.496198][ T228] device_remove_attrs (drivers/base/core.c:2937) [ 312.496375][ T228] ? __pfx_device_remove_attrs (drivers/base/core.c:2937) [ 312.496554][ T228] device_del (drivers/base/core.c:3854) [ 312.496687][ T228] ? __pfx_device_del (drivers/base/core.c:3809) [ 312.496883][ T228] unregister_netdevice_many_notify (net/core/dev.c:2527 (discriminator 3) net/core/dev.c:11568 (discriminator 3)) [ 312.497104][ T228] ? __pfx_unregister_netdevice_many_notify (net/core/dev.c:11472) [ 312.497333][ T228] ? lockdep_hardirqs_on_prepare (kernel/locking/lockdep.c:4347 kernel/locking/lockdep.c:4406) [ 312.497548][ T228] ? __mutex_lock (./arch/x86/include/asm/preempt.h:94 kernel/locking/mutex.c:715 kernel/locking/mutex.c:735) [ 312.497738][ T228] ? nsim_destroy (drivers/net/netdevsim/netdev.c:817) netdevsim [ 312.497938][ T228] ? __pfx___mutex_lock (kernel/locking/mutex.c:734) [ 312.498119][ T228] unregister_netdevice_queue (net/core/dev.c:11467) [ 312.498392][ T228] ? __pfx_unregister_netdevice_queue (net/core/dev.c:11456) [ 312.498657][ T228] nsim_destroy (drivers/net/netdevsim/netdev.c:822 (discriminator 2)) netdevsim [ 312.498843][ T228] __nsim_dev_port_del (drivers/net/netdevsim/dev.c:426 drivers/net/netdevsim/dev.c:1429) netdevsim [ 312.499099][ T228] nsim_dev_reload_destroy (drivers/net/netdevsim/dev.c:1438 drivers/net/netdevsim/dev.c:1661) netdevsim [ 312.499357][ T228] nsim_drv_remove (drivers/net/netdevsim/dev.c:1678) netdevsim [ 312.499551][ T228] device_release_driver_internal (drivers/base/dd.c:1275 drivers/base/dd.c:1296) [ 312.499791][ T228] ? klist_put (lib/klist.c:220) [ 312.499927][ T228] bus_remove_device (./include/linux/kobject.h:193 drivers/base/base.h:73 drivers/base/bus.c:583) [ 312.500113][ T228] device_del (drivers/base/core.c:3855) [ 312.500255][ T228] ? __pfx_device_del (drivers/base/core.c:3809) [ 312.500427][ T228] ? lock_acquire.part.0 (kernel/locking/lockdep.c:467 kernel/locking/lockdep.c:5851) [ 312.500600][ T228] ? kernfs_fop_write_iter (fs/kernfs/file.c:326) [ 312.500826][ T228] device_unregister (drivers/base/core.c:3772 drivers/base/core.c:3896) [ 312.501001][ T228] del_device_store (drivers/net/netdevsim/bus.c:230) netdevsim [ 312.501270][ T228] ? __pfx_del_device_store (drivers/net/netdevsim/bus.c:197) netdevsim [ 312.501515][ T228] ? __pfx_sysfs_kf_write (fs/sysfs/file.c:132) [ 312.501689][ T228] ? sysfs_file_ops (fs/sysfs/file.c:31 (discriminator 1)) [ 312.501895][ T228] ? __pfx_sysfs_kf_write (fs/sysfs/file.c:132) [ 312.502092][ T228] kernfs_fop_write_iter (fs/kernfs/file.c:334) [ 312.502301][ T228] vfs_write (fs/read_write.c:586 fs/read_write.c:679) [ 312.502451][ T228] ? __pfx_vfs_write (fs/read_write.c:660) [ 312.502644][ T228] ? __lock_release (kernel/locking/lockdep.c:5525) [ 312.502824][ T228] ? do_user_addr_fault (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 ./include/linux/mm.h:741 arch/x86/mm/fault.c:1340) [ 312.503015][ T228] ? __pfx___up_read (kernel/locking/rwsem.c:1337) [ 312.503201][ T228] ksys_write (fs/read_write.c:731) [ 312.503357][ T228] ? __pfx_ksys_write (fs/read_write.c:721) [ 312.503536][ T228] ? do_user_addr_fault (./include/linux/rcupdate.h:882 ./include/linux/mm.h:741 arch/x86/mm/fault.c:1340) [ 312.503709][ T228] do_syscall_64 (arch/x86/entry/common.c:52 arch/x86/entry/common.c:83) [ 312.503899][ T228] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:130) [ 312.504130][ T228] RIP: 0033:0x7f954fc2ab77 [ 312.504337][ T228] Code: 0b 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b7 0f 1f 00 f3 0f 1e fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 01 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 51 c3 48 83 ec 28 48 89 54 24 18 48 89 74 24 All code ======== 0: 0b 00 or (%rax),%eax 2: f7 d8 neg %eax 4: 64 89 02 mov %eax,%fs:(%rdx) 7: 48 c7 c0 ff ff ff ff mov $0xffffffffffffffff,%rax e: eb b7 jmp 0xffffffffffffffc7 10: 0f 1f 00 nopl (%rax) 13: f3 0f 1e fa endbr64 17: 64 8b 04 25 18 00 00 mov %fs:0x18,%eax 1e: 00 1f: 85 c0 test %eax,%eax 21: 75 10 jne 0x33 23: b8 01 00 00 00 mov $0x1,%eax 28: 0f 05 syscall 2a:* 48 3d 00 f0 ff ff cmp $0xfffffffffffff000,%rax <-- trapping instruction 30: 77 51 ja 0x83 32: c3 ret 33: 48 83 ec 28 sub $0x28,%rsp 37: 48 89 54 24 18 mov %rdx,0x18(%rsp) 3c: 48 rex.W 3d: 89 .byte 0x89 3e: 74 24 je 0x64 Code starting with the faulting instruction =========================================== 0: 48 3d 00 f0 ff ff cmp $0xfffffffffffff000,%rax 6: 77 51 ja 0x59 8: c3 ret 9: 48 83 ec 28 sub $0x28,%rsp d: 48 89 54 24 18 mov %rdx,0x18(%rsp) 12: 48 rex.W 13: 89 .byte 0x89 14: 74 24 je 0x3a [ 312.504971][ T228] RSP: 002b:00007fffe8f67518 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 [ 312.505256][ T228] RAX: ffffffffffffffda RBX: 0000000000000005 RCX: 00007f954fc2ab77 [ 312.505527][ T228] RDX: 0000000000000005 RSI: 000055d669ed2c90 RDI: 0000000000000001 [ 312.505785][ T228] RBP: 000055d669ed2c90 R08: 0000000000000000 R09: 00007f954fc9d4e0 [ 312.506074][ T228] R10: 00007f954fc9d3e0 R11: 0000000000000246 R12: 0000000000000005 [ 312.506476][ T228] R13: 00007f954fce6760 R14: 0000000000000005 R15: 00007f954fce19c0 | [ 312.505527][ T228] RDX: 0000000000000005 RSI: 000055d669ed2c90 RDI: 0000000000000001 | [ 312.505785][ T228] RBP: 000055d669ed2c90 R08: 0000000000000000 R09: 00007f954fc9d4e0 | [ 312.506074][ T228] R10: 00007f954fc9d3e0 R11: 0000000000000246 R12: 0000000000000005 | [ 312.506476][ T228] R13: 00007f954fce6760 R14: 0000000000000005 R15: 00007f954fce19c0 | [ 312.506760][ T228] [ 493.041424][ T43] INFO: task kworker/u16:0:11 blocked for more than 122 seconds. [ 493.041787][ T43] Not tainted 6.13.0-rc2-virtme #1 [ 493.041963][ T43] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 493.042210][ T43] task:kworker/u16:0 state:D stack:26224 pid:11 tgid:11 ppid:2 flags:0x00004000 [ 493.042547][ T43] Workqueue: netns cleanup_net [ 493.042728][ T43] Call Trace: [ 493.042852][ T43] [ 493.042949][ T43] __schedule (kernel/sched/core.c:5369 kernel/sched/core.c:6756) [ 493.043119][ T43] ? __pfx___schedule (kernel/sched/core.c:6637) [ 493.043282][ T43] ? trace_lock_release (./include/trace/events/lock.h:69 (discriminator 37)) [ 493.043449][ T43] ? trace_irq_enable.constprop.0 (./include/trace/events/preemptirq.h:40) [ 493.043644][ T43] ? trace_lock_release (./include/trace/events/lock.h:69 (discriminator 37)) [ 493.043806][ T43] schedule (./arch/x86/include/asm/preempt.h:84 kernel/sched/core.c:6834 kernel/sched/core.c:6848) [ 493.043927][ T43] schedule_preempt_disabled (./arch/x86/include/asm/preempt.h:79 kernel/sched/core.c:6906) [ 493.044085][ T43] __mutex_lock (kernel/locking/mutex.c:172 kernel/locking/mutex.c:667 kernel/locking/mutex.c:735) [ 493.044239][ T43] ? do_raw_spin_lock (./arch/x86/include/asm/atomic.h:107 ./include/linux/atomic/atomic-arch-fallback.h:2170 ./include/linux/atomic/atomic-instrumented.h:1302 ./include/asm-generic/qspinlock.h:111 kernel/locking/spinlock_debug.c:116) [ 493.044423][ T43] ? default_device_exit_batch (net/core/dev.c:12063) [ 493.044580][ T43] ? __pfx___mutex_lock (kernel/locking/mutex.c:734) [ 493.044743][ T43] ? __wake_up (kernel/sched/wait.c:110 kernel/sched/wait.c:127) [ 493.044872][ T43] ? default_device_exit_batch (net/core/dev.c:12063) [ 493.045021][ T43] default_device_exit_batch (net/core/dev.c:12063) [ 493.045181][ T43] ? __pfx_default_device_exit_batch (net/core/dev.c:12052) [ 493.045382][ T43] ? mutex_is_locked (./arch/x86/include/asm/atomic64_64.h:15 ./include/linux/atomic/atomic-arch-fallback.h:2583 ./include/linux/atomic/atomic-long.h:38 ./include/linux/atomic/atomic-instrumented.h:3189 kernel/locking/mutex.h:47 kernel/locking/mutex.c:66) [ 493.045542][ T43] ? nexthop_net_exit_batch_rtnl (net/ipv4/nexthop.c:585 net/ipv4/nexthop.c:2158 net/ipv4/nexthop.c:2666 net/ipv4/nexthop.c:4010) [ 493.045735][ T43] cleanup_net (net/core/net_namespace.c:647 (discriminator 3)) [ 493.045892][ T43] ? __pfx_lock_acquire.part.0 (kernel/locking/lockdep.c:5814) [ 493.046041][ T43] ? __pfx_cleanup_net (net/core/net_namespace.c:592) [ 493.046196][ T43] ? trace_lock_acquire (./include/trace/events/lock.h:24 (discriminator 37)) [ 493.046349][ T43] ? lock_acquire (kernel/locking/lockdep.c:5822) [ 493.046520][ T43] ? process_one_work (kernel/workqueue.c:3205) [ 493.046689][ T43] process_one_work (kernel/workqueue.c:3229) [ 493.046852][ T43] ? __pfx___lock_release (kernel/locking/lockdep.c:5501) [ 493.047000][ T43] ? __pfx_process_one_work (kernel/workqueue.c:3131) [ 493.047153][ T43] ? assign_work (kernel/workqueue.c:1200) [ 493.047308][ T43] worker_thread (kernel/workqueue.c:3304 kernel/workqueue.c:3391) [ 493.047488][ T43] ? __pfx_worker_thread (kernel/workqueue.c:3337) [ 493.047640][ T43] kthread (kernel/kthread.c:389) [ 493.047767][ T43] ? __pfx_kthread (kernel/kthread.c:342) [ 493.047921][ T43] ret_from_fork (arch/x86/kernel/process.c:147) [ 493.048076][ T43] ? __pfx_kthread (kernel/kthread.c:342) [ 493.048230][ T43] ret_from_fork_asm (arch/x86/entry/entry_64.S:257) | [ 493.047767][ T43] ? __pfx_kthread (kernel/kthread.c:342) | [ 493.047921][ T43] ret_from_fork (arch/x86/kernel/process.c:147) | [ 493.048076][ T43] ? __pfx_kthread (kernel/kthread.c:342) | [ 493.048230][ T43] ret_from_fork_asm (arch/x86/entry/entry_64.S:257) | [ 493.048419][ T43] [ 493.048569][ T43] INFO: task nexthop.sh:228 blocked for more than 122 seconds. [ 493.048784][ T43] Not tainted 6.13.0-rc2-virtme #1 [ 493.048941][ T43] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 493.049217][ T43] task:nexthop.sh state:D stack:24832 pid:228 tgid:228 ppid:227 flags:0x00004002 [ 493.049543][ T43] Call Trace: [ 493.049667][ T43] [ 493.049753][ T43] __schedule (kernel/sched/core.c:5369 kernel/sched/core.c:6756) [ 493.049923][ T43] ? __pfx___schedule (kernel/sched/core.c:6637) [ 493.050089][ T43] ? __pfx_do_raw_spin_lock (kernel/locking/spinlock_debug.c:114) [ 493.050274][ T43] ? trace_lock_release (./include/trace/events/lock.h:69 (discriminator 37)) [ 493.050470][ T43] schedule (./arch/x86/include/asm/preempt.h:84 kernel/sched/core.c:6834 kernel/sched/core.c:6848) [ 493.050597][ T43] kernfs_drain (fs/kernfs/dir.c:505 (discriminator 9)) [ 493.050759][ T43] ? __pfx_kernfs_drain (fs/kernfs/dir.c:481) [ 493.050918][ T43] ? __pfx_autoremove_wake_function (kernel/sched/wait.c:383) [ 493.051119][ T43] __kernfs_remove (fs/kernfs/dir.c:1492) [ 493.051282][ T43] kernfs_remove_by_name_ns (fs/kernfs/dir.c:1695) [ 493.051456][ T43] remove_files (fs/sysfs/group.c:27 (discriminator 3)) [ 493.051610][ T43] sysfs_remove_group (fs/sysfs/group.c:323) [ 493.051762][ T43] sysfs_remove_groups (fs/sysfs/group.c:345 fs/sysfs/group.c:338) [ 493.051909][ T43] device_remove_attrs (drivers/base/core.c:2937) [ 493.052058][ T43] ? __pfx_device_remove_attrs (drivers/base/core.c:2937) [ 493.052211][ T43] device_del (drivers/base/core.c:3854) [ 493.052331][ T43] ? __pfx_device_del (drivers/base/core.c:3809) [ 493.052506][ T43] unregister_netdevice_many_notify (net/core/dev.c:2527 (discriminator 3) net/core/dev.c:11568 (discriminator 3)) [ 493.052705][ T43] ? __pfx_unregister_netdevice_many_notify (net/core/dev.c:11472) [ 493.052889][ T43] ? lockdep_hardirqs_on_prepare (kernel/locking/lockdep.c:4347 kernel/locking/lockdep.c:4406) [ 493.053070][ T43] ? __mutex_lock (./arch/x86/include/asm/preempt.h:94 kernel/locking/mutex.c:715 kernel/locking/mutex.c:735) [ 493.053221][ T43] ? nsim_destroy (drivers/net/netdevsim/netdev.c:817) netdevsim [ 493.053394][ T43] ? __pfx___mutex_lock (kernel/locking/mutex.c:734) [ 493.053546][ T43] unregister_netdevice_queue (net/core/dev.c:11467) [ 493.053703][ T43] ? __pfx_unregister_netdevice_queue (net/core/dev.c:11456) [ 493.053889][ T43] nsim_destroy (drivers/net/netdevsim/netdev.c:822 (discriminator 2)) netdevsim [ 493.054084][ T43] __nsim_dev_port_del (drivers/net/netdevsim/dev.c:426 drivers/net/netdevsim/dev.c:1429) netdevsim [ 493.054318][ T43] nsim_dev_reload_destroy (drivers/net/netdevsim/dev.c:1438 drivers/net/netdevsim/dev.c:1661) netdevsim [ 493.054531][ T43] nsim_drv_remove (drivers/net/netdevsim/dev.c:1678) netdevsim [ 493.054696][ T43] device_release_driver_internal (drivers/base/dd.c:1275 drivers/base/dd.c:1296) [ 493.054887][ T43] ? klist_put (lib/klist.c:220) [ 493.055011][ T43] bus_remove_device (./include/linux/kobject.h:193 drivers/base/base.h:73 drivers/base/bus.c:583) [ 493.055160][ T43] device_del (drivers/base/core.c:3855) [ 493.055284][ T43] ? __pfx_device_del (drivers/base/core.c:3809) [ 493.055456][ T43] ? lock_acquire.part.0 (kernel/locking/lockdep.c:467 kernel/locking/lockdep.c:5851) [ 493.055604][ T43] ? kernfs_fop_write_iter (fs/kernfs/file.c:326) [ 493.055755][ T43] device_unregister (drivers/base/core.c:3772 drivers/base/core.c:3896) [ 493.055900][ T43] del_device_store (drivers/net/netdevsim/bus.c:230) netdevsim [ 493.056102][ T43] ? __pfx_del_device_store (drivers/net/netdevsim/bus.c:197) netdevsim [ 493.056291][ T43] ? __pfx_sysfs_kf_write (fs/sysfs/file.c:132) [ 493.056465][ T43] ? sysfs_file_ops (fs/sysfs/file.c:31 (discriminator 1)) [ 493.056617][ T43] ? __pfx_sysfs_kf_write (fs/sysfs/file.c:132) [ 493.056766][ T43] kernfs_fop_write_iter (fs/kernfs/file.c:334) [ 493.056917][ T43] vfs_write (fs/read_write.c:586 fs/read_write.c:679) [ 493.057038][ T43] ? __pfx_vfs_write (fs/read_write.c:660) [ 493.057195][ T43] ? __lock_release (kernel/locking/lockdep.c:5525) [ 493.057343][ T43] ? do_user_addr_fault (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 ./include/linux/mm.h:741 arch/x86/mm/fault.c:1340) [ 493.057506][ T43] ? __pfx___up_read (kernel/locking/rwsem.c:1337) [ 493.057657][ T43] ksys_write (fs/read_write.c:731) [ 493.057774][ T43] ? __pfx_ksys_write (fs/read_write.c:721) [ 493.057920][ T43] ? do_user_addr_fault (./include/linux/rcupdate.h:882 ./include/linux/mm.h:741 arch/x86/mm/fault.c:1340) [ 493.058072][ T43] do_syscall_64 (arch/x86/entry/common.c:52 arch/x86/entry/common.c:83) [ 493.058233][ T43] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:130) [ 493.058440][ T43] RIP: 0033:0x7f954fc2ab77 [ 493.058641][ T43] RSP: 002b:00007fffe8f67518 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 [ 493.058884][ T43] RAX: ffffffffffffffda RBX: 0000000000000005 RCX: 00007f954fc2ab77 [ 493.059122][ T43] RDX: 0000000000000005 RSI: 000055d669ed2c90 RDI: 0000000000000001 [ 493.059360][ T43] RBP: 000055d669ed2c90 R08: 0000000000000000 R09: 00007f954fc9d4e0 [ 493.059578][ T43] R10: 00007f954fc9d3e0 R11: 0000000000000246 R12: 0000000000000005 Finger prints: __schedule:schedule:schedule_preempt_disabled:__mutex_lock:default_device_exit_batch ret_from_fork:ret_from_fork_asm:__schedule:schedule:kernfs_drain print_circular_bug:check_noncircular:check_prev_add:validate_chain:__lock_acquire