====================================== | [ 36.350481][ T289] other info that might help us debug this: | [ 36.350665][ T289] no locks held by ip/289. | [ 36.350795][ T289] | [ 36.350795][ T289] stack backtrace: [ 36.351149][ T289] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 36.351436][ T289] Call Trace: [ 36.351538][ T289] [ 36.351604][ T289] dump_stack_lvl (lib/dump_stack.c:123) [ 36.351736][ T289] ? __rtnl_unlock (net/core/rtnetlink.c:140 (discriminator 3)) [ 36.351865][ T289] print_unlock_imbalance_bug (kernel/locking/lockdep.c:5266) [ 36.351992][ T289] __lock_release (kernel/locking/lockdep.c:5502) [ 36.352119][ T289] ? __pfx___lock_release (kernel/locking/lockdep.c:5477) [ 36.352249][ T289] ? __rtnl_unlock (net/core/rtnetlink.c:140 (discriminator 3)) [ 36.352379][ T289] lock_release (kernel/locking/lockdep.c:467 kernel/locking/lockdep.c:5848) [ 36.352507][ T289] __mutex_unlock_slowpath (./include/linux/instrumented.h:68 ./include/linux/atomic/atomic-instrumented.h:3188 kernel/locking/mutex.c:921) [ 36.352635][ T289] ? __pfx_lock_acquire.part.0 (kernel/locking/lockdep.c:5790) [ 36.352762][ T289] ? __pfx___mutex_unlock_slowpath (kernel/locking/mutex.c:907) [ 36.352923][ T289] ? rtnl_link_ops_get (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/core/rtnetlink.c:558) [ 36.353048][ T289] __rtnl_unlock (net/core/rtnetlink.c:140 (discriminator 3)) [ 36.353174][ T289] rtnl_newlink (net/core/rtnetlink.c:3961) [ 36.353306][ T289] ? __pfx_rtnl_newlink (net/core/rtnetlink.c:3919) [ 36.353433][ T289] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 36.353560][ T289] ? __lock_release (kernel/locking/lockdep.c:5501) [ 36.353683][ T289] ? rtnetlink_rcv_msg (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/core/rtnetlink.c:6926) [ 36.353810][ T289] ? __pfx___lock_release (kernel/locking/lockdep.c:5477) [ 36.353939][ T289] ? rtnetlink_rcv_msg (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/core/rtnetlink.c:6926) [ 36.354065][ T289] ? __pfx_rtnl_newlink (net/core/rtnetlink.c:3919) [ 36.354189][ T289] rtnetlink_rcv_msg (net/core/rtnetlink.c:6928) [ 36.354316][ T289] ? __pfx_rtnetlink_rcv_msg (net/core/rtnetlink.c:6831) [ 36.354443][ T289] ? hlock_class (./arch/x86/include/asm/bitops.h:227 ./arch/x86/include/asm/bitops.h:239 ./include/asm-generic/bitops/instrumented-non-atomic.h:142 kernel/locking/lockdep.c:228) [ 36.354570][ T289] ? mark_lock (kernel/locking/lockdep.c:4703 (discriminator 3)) [ 36.354681][ T289] ? __lock_acquire (kernel/locking/lockdep.c:5202) [ 36.354808][ T289] netlink_rcv_skb (net/netlink/af_netlink.c:2536) [ 36.354935][ T289] ? __pfx_rtnetlink_rcv_msg (net/core/rtnetlink.c:6831) [ 36.355062][ T289] ? __pfx_netlink_rcv_skb (net/netlink/af_netlink.c:2513) [ 36.355189][ T289] ? netlink_deliver_tap (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/netlink/af_netlink.c:340) [ 36.355317][ T289] ? netlink_deliver_tap (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 ./include/net/netns/generic.h:48 net/netlink/af_netlink.c:333) [ 36.355444][ T289] netlink_unicast (net/netlink/af_netlink.c:1316 net/netlink/af_netlink.c:1342) [ 36.355569][ T289] ? __pfx_netlink_unicast (net/netlink/af_netlink.c:1327) [ 36.355695][ T289] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 36.355822][ T289] netlink_sendmsg (net/netlink/af_netlink.c:1886) [ 36.355951][ T289] ? __pfx_netlink_sendmsg (net/netlink/af_netlink.c:1805) [ 36.356078][ T289] ____sys_sendmsg (net/socket.c:729 net/socket.c:744 net/socket.c:2609) [ 36.356206][ T289] ? __pfx_____sys_sendmsg (net/socket.c:2555) [ 36.356337][ T289] ? __pfx_copy_msghdr_from_user (net/socket.c:2535) [ 36.356494][ T289] ? hlock_class (./arch/x86/include/asm/bitops.h:227 ./arch/x86/include/asm/bitops.h:239 ./include/asm-generic/bitops/instrumented-non-atomic.h:142 kernel/locking/lockdep.c:228) [ 36.356620][ T289] ? validate_chain (./include/linux/hash.h:78 kernel/locking/lockdep.c:3794 kernel/locking/lockdep.c:3817 kernel/locking/lockdep.c:3872) [ 36.356844][ T289] ___sys_sendmsg (net/socket.c:2665) [ 36.356976][ T289] ? __pfx_validate_chain (kernel/locking/lockdep.c:3860) [ 36.357103][ T289] ? __pfx____sys_sendmsg (net/socket.c:2652) [ 36.357228][ T289] ? __lock_acquire (kernel/locking/lockdep.c:5202) [ 36.357357][ T289] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 36.357482][ T289] ? __lock_release (kernel/locking/lockdep.c:5501) [ 36.357604][ T289] ? __might_fault (mm/memory.c:6716 mm/memory.c:6709) [ 36.357735][ T289] ? fdget (./include/linux/atomic/atomic-arch-fallback.h:479 ./include/linux/atomic/atomic-instrumented.h:50 fs/file.c:1114 fs/file.c:1128) [ 36.357828][ T289] ? lock_acquire (kernel/locking/lockdep.c:5798) [ 36.357954][ T289] __sys_sendmsg (./include/linux/file.h:35 net/socket.c:2694) [ 36.358078][ T289] ? __pfx___sys_sendmsg (net/socket.c:2680) [ 36.358203][ T289] ? rseq_update_cpu_node_id (kernel/rseq.c:109 (discriminator 10)) [ 36.358334][ T289] do_syscall_64 (arch/x86/entry/common.c:52 arch/x86/entry/common.c:83) [ 36.358463][ T289] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:130) [ 36.358617][ T289] RIP: 0033:0x7f5ad69a07b7 [ 36.358747][ T289] Code: 0a 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b9 0f 1f 00 f3 0f 1e fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 2e 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 51 c3 48 83 ec 28 89 54 24 1c 48 89 74 24 10 All code ======== 0: 0a 00 or (%rax),%al 2: f7 d8 neg %eax 4: 64 89 02 mov %eax,%fs:(%rdx) 7: 48 c7 c0 ff ff ff ff mov $0xffffffffffffffff,%rax e: eb b9 jmp 0xffffffffffffffc9 10: 0f 1f 00 nopl (%rax) 13: f3 0f 1e fa endbr64 17: 64 8b 04 25 18 00 00 mov %fs:0x18,%eax 1e: 00 1f: 85 c0 test %eax,%eax 21: 75 10 jne 0x33 23: b8 2e 00 00 00 mov $0x2e,%eax 28: 0f 05 syscall 2a:* 48 3d 00 f0 ff ff cmp $0xfffffffffffff000,%rax <-- trapping instruction 30: 77 51 ja 0x83 32: c3 ret 33: 48 83 ec 28 sub $0x28,%rsp 37: 89 54 24 1c mov %edx,0x1c(%rsp) 3b: 48 89 74 24 10 mov %rsi,0x10(%rsp) Code starting with the faulting instruction =========================================== 0: 48 3d 00 f0 ff ff cmp $0xfffffffffffff000,%rax 6: 77 51 ja 0x59 8: c3 ret 9: 48 83 ec 28 sub $0x28,%rsp d: 89 54 24 1c mov %edx,0x1c(%rsp) 11: 48 89 74 24 10 mov %rsi,0x10(%rsp) [ 36.359187][ T289] RSP: 002b:00007ffda1a67088 EFLAGS: 00000246 ORIG_RAX: 000000000000002e [ 36.359378][ T289] RAX: ffffffffffffffda RBX: 00007ffda1a677c0 RCX: 00007f5ad69a07b7 [ 36.359564][ T289] RDX: 0000000000000000 RSI: 00007ffda1a670f0 RDI: 0000000000000005 [ 36.359748][ T289] RBP: 0000000000000002 R08: 0000000000000040 R09: 0000000000000000 [ 36.359931][ T289] R10: 00007f5ad69efac0 R11: 0000000000000246 R12: 00007ffda1a677d8 [ 36.360119][ T289] R13: 000000006729b78c R14: 0000000000496600 R15: 00007ffda1a671d4 | WAIT TIMEOUT stderr | | Ctrl-C stderr | | Ctrl-C stderr [ 247.089371][ T43] INFO: task ip:289 blocked for more than 122 seconds. [ 247.089750][ T43] Not tainted 6.12.0-rc5-virtme #1 [ 247.089885][ T43] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 247.090103][ T43] task:ip state:D stack:25768 pid:289 tgid:289 ppid:237 flags:0x00004006 [ 247.090388][ T43] Call Trace: [ 247.090491][ T43] [ 247.090563][ T43] __schedule (kernel/sched/core.c:5328 kernel/sched/core.c:6690) [ 247.090704][ T43] ? __pfx___schedule (kernel/sched/core.c:6546) [ 247.090839][ T43] ? trace_lock_acquire (./include/trace/events/lock.h:24 (discriminator 52)) [ 247.090974][ T43] ? trace_lock_release (./include/trace/events/lock.h:69 (discriminator 52)) [ 247.091106][ T43] schedule (./arch/x86/include/asm/preempt.h:84 kernel/sched/core.c:6768 kernel/sched/core.c:6782) [ 247.091208][ T43] schedule_preempt_disabled (./arch/x86/include/asm/preempt.h:79 kernel/sched/core.c:6840) [ 247.091353][ T43] __mutex_lock (kernel/locking/mutex.c:197 kernel/locking/mutex.c:686 kernel/locking/mutex.c:752) [ 247.091506][ T43] ? rtnl_newlink (net/core/rtnetlink.c:321 net/core/rtnetlink.c:4025) [ 247.091639][ T43] ? __pfx___mutex_lock (kernel/locking/mutex.c:751) [ 247.091768][ T43] ? trace_lock_acquire (./include/trace/events/lock.h:24 (discriminator 52)) [ 247.091902][ T43] ? rtnl_newlink (net/core/rtnetlink.c:321 net/core/rtnetlink.c:4025) [ 247.092029][ T43] rtnl_newlink (net/core/rtnetlink.c:321 net/core/rtnetlink.c:4025) [ 247.092162][ T43] ? __pfx_rtnl_newlink (net/core/rtnetlink.c:3919) [ 247.092303][ T43] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 247.092435][ T43] ? __pfx___lock_release (kernel/locking/lockdep.c:5477) [ 247.092573][ T43] ? rtnetlink_rcv_msg (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/core/rtnetlink.c:6926) [ 247.092701][ T43] ? __pfx_rtnl_newlink (net/core/rtnetlink.c:3919) [ 247.092829][ T43] rtnetlink_rcv_msg (net/core/rtnetlink.c:6928) [ 247.092959][ T43] ? __pfx_rtnetlink_rcv_msg (net/core/rtnetlink.c:6831) [ 247.093091][ T43] ? hlock_class (./arch/x86/include/asm/bitops.h:227 ./arch/x86/include/asm/bitops.h:239 ./include/asm-generic/bitops/instrumented-non-atomic.h:142 kernel/locking/lockdep.c:228) [ 247.093219][ T43] ? mark_lock (kernel/locking/lockdep.c:4703 (discriminator 3)) [ 247.093336][ T43] ? __lock_acquire (kernel/locking/lockdep.c:5202) [ 247.093465][ T43] netlink_rcv_skb (net/netlink/af_netlink.c:2536) [ 247.093599][ T43] ? __pfx_rtnetlink_rcv_msg (net/core/rtnetlink.c:6831) [ 247.093730][ T43] ? __pfx_netlink_rcv_skb (net/netlink/af_netlink.c:2513) [ 247.093862][ T43] ? netlink_deliver_tap (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/netlink/af_netlink.c:340) [ 247.093993][ T43] ? netlink_deliver_tap (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 ./include/net/netns/generic.h:48 net/netlink/af_netlink.c:333) [ 247.094122][ T43] netlink_unicast (net/netlink/af_netlink.c:1316 net/netlink/af_netlink.c:1342) [ 247.094261][ T43] ? __pfx_netlink_unicast (net/netlink/af_netlink.c:1327) [ 247.094387][ T43] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 247.094516][ T43] netlink_sendmsg (net/netlink/af_netlink.c:1886) [ 247.094646][ T43] ? __pfx_netlink_sendmsg (net/netlink/af_netlink.c:1805) [ 247.094782][ T43] ____sys_sendmsg (net/socket.c:729 net/socket.c:744 net/socket.c:2609) [ 247.094911][ T43] ? __pfx_____sys_sendmsg (net/socket.c:2555) [ 247.095040][ T43] ? __pfx_copy_msghdr_from_user (net/socket.c:2535) [ 247.095201][ T43] ? hlock_class (./arch/x86/include/asm/bitops.h:227 ./arch/x86/include/asm/bitops.h:239 ./include/asm-generic/bitops/instrumented-non-atomic.h:142 kernel/locking/lockdep.c:228) [ 247.095346][ T43] ? validate_chain (./include/linux/hash.h:78 kernel/locking/lockdep.c:3794 kernel/locking/lockdep.c:3817 kernel/locking/lockdep.c:3872) [ 247.095477][ T43] ___sys_sendmsg (net/socket.c:2665) [ 247.095609][ T43] ? __pfx_validate_chain (kernel/locking/lockdep.c:3860) [ 247.095739][ T43] ? __pfx____sys_sendmsg (net/socket.c:2652) [ 247.095877][ T43] ? __lock_acquire (kernel/locking/lockdep.c:5202) [ 247.096009][ T43] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 247.096140][ T43] ? __lock_release (kernel/locking/lockdep.c:5501) [ 247.096274][ T43] ? __might_fault (mm/memory.c:6716 mm/memory.c:6709) [ 247.096408][ T43] ? fdget (./include/linux/atomic/atomic-arch-fallback.h:479 ./include/linux/atomic/atomic-instrumented.h:50 fs/file.c:1114 fs/file.c:1128) [ 247.096517][ T43] ? lock_acquire (kernel/locking/lockdep.c:5798) [ 247.096649][ T43] __sys_sendmsg (./include/linux/file.h:35 net/socket.c:2694) [ 247.096781][ T43] ? __pfx___sys_sendmsg (net/socket.c:2680) [ 247.096914][ T43] ? rseq_update_cpu_node_id (kernel/rseq.c:109 (discriminator 10)) [ 247.097051][ T43] do_syscall_64 (arch/x86/entry/common.c:52 arch/x86/entry/common.c:83) [ 247.097181][ T43] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:130) [ 247.097351][ T43] RIP: 0033:0x7f5ad69a07b7 [ 247.097489][ T43] RSP: 002b:00007ffda1a67088 EFLAGS: 00000246 ORIG_RAX: 000000000000002e [ 247.097686][ T43] RAX: ffffffffffffffda RBX: 00007ffda1a677c0 RCX: 00007f5ad69a07b7 [ 247.097879][ T43] RDX: 0000000000000000 RSI: 00007ffda1a670f0 RDI: 0000000000000005 [ 247.098075][ T43] RBP: 0000000000000002 R08: 0000000000000040 R09: 0000000000000000 [ 247.098274][ T43] R10: 00007f5ad69efac0 R11: 0000000000000246 R12: 00007ffda1a677d8 Finger prints: __schedule:schedule:schedule_preempt_disabled:__mutex_lock:rtnl_newlink print_unlock_imbalance_bug:__lock_release:lock_release:__mutex_unlock_slowpath:__rtnl_unlock