====================================== | [ 33.170507][ T266] other info that might help us debug this: | [ 33.170707][ T266] no locks held by ip/266. | [ 33.170843][ T266] | [ 33.170843][ T266] stack backtrace: [ 33.171217][ T266] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 33.171529][ T266] Call Trace: [ 33.171645][ T266] [ 33.171715][ T266] dump_stack_lvl (lib/dump_stack.c:123) [ 33.171855][ T266] ? __rtnl_unlock (net/core/rtnetlink.c:140 (discriminator 3)) [ 33.171991][ T266] print_unlock_imbalance_bug (kernel/locking/lockdep.c:5266) [ 33.172127][ T266] __lock_release (kernel/locking/lockdep.c:5502) [ 33.172261][ T266] ? __pfx___lock_release (kernel/locking/lockdep.c:5477) [ 33.172396][ T266] ? __rtnl_unlock (net/core/rtnetlink.c:140 (discriminator 3)) [ 33.172531][ T266] lock_release (kernel/locking/lockdep.c:467 kernel/locking/lockdep.c:5848) [ 33.172665][ T266] __mutex_unlock_slowpath (./include/linux/instrumented.h:68 ./include/linux/atomic/atomic-instrumented.h:3188 kernel/locking/mutex.c:921) [ 33.172798][ T266] ? __pfx_lock_acquire.part.0 (kernel/locking/lockdep.c:5790) [ 33.172932][ T266] ? __pfx___mutex_unlock_slowpath (kernel/locking/mutex.c:907) [ 33.173101][ T266] ? rtnl_link_ops_get (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/core/rtnetlink.c:558) [ 33.173257][ T266] __rtnl_unlock (net/core/rtnetlink.c:140 (discriminator 3)) [ 33.173397][ T266] rtnl_newlink (net/core/rtnetlink.c:3961) [ 33.173536][ T266] ? __pfx_rtnl_newlink (net/core/rtnetlink.c:3919) [ 33.173675][ T266] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 33.173818][ T266] ? __lock_release (kernel/locking/lockdep.c:5501) [ 33.173956][ T266] ? rtnetlink_rcv_msg (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/core/rtnetlink.c:6926) [ 33.174095][ T266] ? __pfx___lock_release (kernel/locking/lockdep.c:5477) [ 33.174235][ T266] ? rtnetlink_rcv_msg (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/core/rtnetlink.c:6926) [ 33.174383][ T266] ? __pfx_rtnl_newlink (net/core/rtnetlink.c:3919) [ 33.174515][ T266] rtnetlink_rcv_msg (net/core/rtnetlink.c:6928) [ 33.174652][ T266] ? __pfx_rtnetlink_rcv_msg (net/core/rtnetlink.c:6831) [ 33.174785][ T266] ? hlock_class (./arch/x86/include/asm/bitops.h:227 ./arch/x86/include/asm/bitops.h:239 ./include/asm-generic/bitops/instrumented-non-atomic.h:142 kernel/locking/lockdep.c:228) [ 33.174923][ T266] ? mark_lock (kernel/locking/lockdep.c:4703 (discriminator 3)) [ 33.175025][ T266] ? __lock_acquire (kernel/locking/lockdep.c:5202) [ 33.175161][ T266] netlink_rcv_skb (net/netlink/af_netlink.c:2536) [ 33.175297][ T266] ? __pfx_rtnetlink_rcv_msg (net/core/rtnetlink.c:6831) [ 33.175429][ T266] ? __pfx_netlink_rcv_skb (net/netlink/af_netlink.c:2513) [ 33.175566][ T266] ? netlink_deliver_tap (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/netlink/af_netlink.c:340) [ 33.175699][ T266] ? netlink_deliver_tap (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 ./include/net/netns/generic.h:48 net/netlink/af_netlink.c:333) [ 33.175835][ T266] netlink_unicast (net/netlink/af_netlink.c:1316 net/netlink/af_netlink.c:1342) [ 33.175972][ T266] ? __pfx_netlink_unicast (net/netlink/af_netlink.c:1327) [ 33.176106][ T266] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 33.176244][ T266] ? __lock_release (kernel/locking/lockdep.c:5501) [ 33.176382][ T266] netlink_sendmsg (net/netlink/af_netlink.c:1886) [ 33.176543][ T266] ? __pfx_netlink_sendmsg (net/netlink/af_netlink.c:1805) [ 33.176685][ T266] ? _copy_from_user (./arch/x86/include/asm/smap.h:33 ./arch/x86/include/asm/uaccess_64.h:134 ./arch/x86/include/asm/uaccess_64.h:141 ./include/linux/uaccess.h:178 lib/usercopy.c:18) [ 33.176830][ T266] ____sys_sendmsg (net/socket.c:729 net/socket.c:744 net/socket.c:2609) [ 33.176973][ T266] ? __pfx_____sys_sendmsg (net/socket.c:2555) [ 33.177125][ T266] ? __pfx_copy_msghdr_from_user (net/socket.c:2535) [ 33.177303][ T266] ? __pfx_validate_chain (kernel/locking/lockdep.c:3860) [ 33.177443][ T266] ? hlock_class (./arch/x86/include/asm/bitops.h:227 ./arch/x86/include/asm/bitops.h:239 ./include/asm-generic/bitops/instrumented-non-atomic.h:142 kernel/locking/lockdep.c:228) [ 33.177584][ T266] ___sys_sendmsg (net/socket.c:2665) [ 33.177735][ T266] ? __pfx____sys_sendmsg (net/socket.c:2652) [ 33.177887][ T266] ? __lock_acquire (kernel/locking/lockdep.c:5202) [ 33.178030][ T266] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 33.178170][ T266] ? __lock_release (kernel/locking/lockdep.c:5501) [ 33.178314][ T266] ? __might_fault (mm/memory.c:6716 mm/memory.c:6709) [ 33.178468][ T266] ? __pfx___lock_release (kernel/locking/lockdep.c:5477) [ 33.178620][ T266] ? trace_lock_acquire (./include/trace/events/lock.h:24 (discriminator 52)) [ 33.178759][ T266] ? __might_fault (mm/memory.c:6716 mm/memory.c:6709) [ 33.178900][ T266] ? lock_acquire (kernel/locking/lockdep.c:5798) [ 33.179039][ T266] ? fdget (./include/linux/atomic/atomic-arch-fallback.h:479 ./include/linux/atomic/atomic-instrumented.h:50 fs/file.c:1114 fs/file.c:1128) [ 33.179148][ T266] __sys_sendmsg (./include/linux/file.h:35 net/socket.c:2694) [ 33.179287][ T266] ? __pfx___sys_sendmsg (net/socket.c:2680) [ 33.179426][ T266] ? rseq_update_cpu_node_id (kernel/rseq.c:109 (discriminator 10)) [ 33.179580][ T266] ? __pfx___rseq_handle_notify_resume (kernel/rseq.c:316) [ 33.179769][ T266] do_syscall_64 (arch/x86/entry/common.c:52 arch/x86/entry/common.c:83) [ 33.179909][ T266] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:130) [ 33.180094][ T266] RIP: 0033:0x7f217acca7b7 [ 33.180238][ T266] Code: 0a 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b9 0f 1f 00 f3 0f 1e fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 2e 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 51 c3 48 83 ec 28 89 54 24 1c 48 89 74 24 10 All code ======== 0: 0a 00 or (%rax),%al 2: f7 d8 neg %eax 4: 64 89 02 mov %eax,%fs:(%rdx) 7: 48 c7 c0 ff ff ff ff mov $0xffffffffffffffff,%rax e: eb b9 jmp 0xffffffffffffffc9 10: 0f 1f 00 nopl (%rax) 13: f3 0f 1e fa endbr64 17: 64 8b 04 25 18 00 00 mov %fs:0x18,%eax 1e: 00 1f: 85 c0 test %eax,%eax 21: 75 10 jne 0x33 23: b8 2e 00 00 00 mov $0x2e,%eax 28: 0f 05 syscall 2a:* 48 3d 00 f0 ff ff cmp $0xfffffffffffff000,%rax <-- trapping instruction 30: 77 51 ja 0x83 32: c3 ret 33: 48 83 ec 28 sub $0x28,%rsp 37: 89 54 24 1c mov %edx,0x1c(%rsp) 3b: 48 89 74 24 10 mov %rsi,0x10(%rsp) Code starting with the faulting instruction =========================================== 0: 48 3d 00 f0 ff ff cmp $0xfffffffffffff000,%rax 6: 77 51 ja 0x59 8: c3 ret 9: 48 83 ec 28 sub $0x28,%rsp d: 89 54 24 1c mov %edx,0x1c(%rsp) 11: 48 89 74 24 10 mov %rsi,0x10(%rsp) [ 33.180726][ T266] RSP: 002b:00007ffd31957c18 EFLAGS: 00000246 ORIG_RAX: 000000000000002e [ 33.180943][ T266] RAX: ffffffffffffffda RBX: 00007ffd31958340 RCX: 00007f217acca7b7 [ 33.181154][ T266] RDX: 0000000000000000 RSI: 00007ffd31957c80 RDI: 0000000000000005 [ 33.181429][ T266] RBP: 0000000000000005 R08: 000000000000006c R09: 00007ffd319569f0 [ 33.181632][ T266] R10: 00007f217ab83708 R11: 0000000000000246 R12: 00007ffd31958368 [ 33.181837][ T266] R13: 000000006729dd8d R14: 0000000000496600 R15: 00007ffd31957d70 | [ 33.181154][ T266] RDX: 0000000000000000 RSI: 00007ffd31957c80 RDI: 0000000000000005 | [ 33.181429][ T266] RBP: 0000000000000005 R08: 000000000000006c R09: 00007ffd319569f0 | [ 33.181632][ T266] R10: 00007f217ab83708 R11: 0000000000000246 R12: 00007ffd31958368 | [ 33.181837][ T266] R13: 000000006729dd8d R14: 0000000000496600 R15: 00007ffd31957d70 | [ 33.182112][ T266] [ 247.069228][ T44] INFO: task ip:266 blocked for more than 122 seconds. [ 247.069719][ T44] Not tainted 6.12.0-rc5-virtme #1 [ 247.069867][ T44] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 247.070113][ T44] task:ip state:D stack:25648 pid:266 tgid:266 ppid:245 flags:0x00004002 [ 247.070377][ T44] Call Trace: [ 247.070486][ T44] [ 247.070559][ T44] __schedule (kernel/sched/core.c:5328 kernel/sched/core.c:6690) [ 247.070699][ T44] ? __pfx___schedule (kernel/sched/core.c:6546) [ 247.070847][ T44] ? trace_lock_acquire (./include/trace/events/lock.h:24 (discriminator 52)) [ 247.070986][ T44] ? trace_lock_release (./include/trace/events/lock.h:69 (discriminator 52)) [ 247.071138][ T44] schedule (./arch/x86/include/asm/preempt.h:84 kernel/sched/core.c:6768 kernel/sched/core.c:6782) [ 247.071246][ T44] schedule_preempt_disabled (./arch/x86/include/asm/preempt.h:79 kernel/sched/core.c:6840) [ 247.071398][ T44] __mutex_lock (kernel/locking/mutex.c:197 kernel/locking/mutex.c:686 kernel/locking/mutex.c:752) [ 247.071547][ T44] ? rtnl_newlink (net/core/rtnetlink.c:321 net/core/rtnetlink.c:4025) [ 247.071679][ T44] ? __pfx___mutex_lock (kernel/locking/mutex.c:751) [ 247.071813][ T44] ? trace_lock_acquire (./include/trace/events/lock.h:24 (discriminator 52)) [ 247.071946][ T44] ? rtnl_newlink (net/core/rtnetlink.c:321 net/core/rtnetlink.c:4025) [ 247.072077][ T44] rtnl_newlink (net/core/rtnetlink.c:321 net/core/rtnetlink.c:4025) [ 247.072215][ T44] ? __pfx_rtnl_newlink (net/core/rtnetlink.c:3919) [ 247.072348][ T44] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 247.072477][ T44] ? __pfx___lock_release (kernel/locking/lockdep.c:5477) [ 247.072616][ T44] ? rtnetlink_rcv_msg (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/core/rtnetlink.c:6926) [ 247.072743][ T44] ? __pfx_rtnl_newlink (net/core/rtnetlink.c:3919) [ 247.072872][ T44] rtnetlink_rcv_msg (net/core/rtnetlink.c:6928) [ 247.073000][ T44] ? __pfx_rtnetlink_rcv_msg (net/core/rtnetlink.c:6831) [ 247.073134][ T44] ? hlock_class (./arch/x86/include/asm/bitops.h:227 ./arch/x86/include/asm/bitops.h:239 ./include/asm-generic/bitops/instrumented-non-atomic.h:142 kernel/locking/lockdep.c:228) [ 247.073262][ T44] ? mark_lock (kernel/locking/lockdep.c:4703 (discriminator 3)) [ 247.073360][ T44] ? __lock_acquire (kernel/locking/lockdep.c:5202) [ 247.073489][ T44] netlink_rcv_skb (net/netlink/af_netlink.c:2536) [ 247.073629][ T44] ? __pfx_rtnetlink_rcv_msg (net/core/rtnetlink.c:6831) [ 247.073755][ T44] ? __pfx_netlink_rcv_skb (net/netlink/af_netlink.c:2513) [ 247.073887][ T44] ? netlink_deliver_tap (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/netlink/af_netlink.c:340) [ 247.074016][ T44] ? netlink_deliver_tap (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 ./include/net/netns/generic.h:48 net/netlink/af_netlink.c:333) [ 247.074152][ T44] netlink_unicast (net/netlink/af_netlink.c:1316 net/netlink/af_netlink.c:1342) [ 247.074283][ T44] ? __pfx_netlink_unicast (net/netlink/af_netlink.c:1327) [ 247.074409][ T44] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 247.074538][ T44] ? __lock_release (kernel/locking/lockdep.c:5501) [ 247.074670][ T44] netlink_sendmsg (net/netlink/af_netlink.c:1886) [ 247.074801][ T44] ? __pfx_netlink_sendmsg (net/netlink/af_netlink.c:1805) [ 247.074931][ T44] ? _copy_from_user (./arch/x86/include/asm/smap.h:33 ./arch/x86/include/asm/uaccess_64.h:134 ./arch/x86/include/asm/uaccess_64.h:141 ./include/linux/uaccess.h:178 lib/usercopy.c:18) [ 247.075062][ T44] ____sys_sendmsg (net/socket.c:729 net/socket.c:744 net/socket.c:2609) [ 247.075226][ T44] ? __pfx_____sys_sendmsg (net/socket.c:2555) [ 247.075359][ T44] ? __pfx_copy_msghdr_from_user (net/socket.c:2535) [ 247.075528][ T44] ? __pfx_validate_chain (kernel/locking/lockdep.c:3860) [ 247.075662][ T44] ? hlock_class (./arch/x86/include/asm/bitops.h:227 ./arch/x86/include/asm/bitops.h:239 ./include/asm-generic/bitops/instrumented-non-atomic.h:142 kernel/locking/lockdep.c:228) [ 247.075798][ T44] ___sys_sendmsg (net/socket.c:2665) [ 247.075932][ T44] ? __pfx____sys_sendmsg (net/socket.c:2652) [ 247.076064][ T44] ? __lock_acquire (kernel/locking/lockdep.c:5202) [ 247.076202][ T44] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 247.076336][ T44] ? __lock_release (kernel/locking/lockdep.c:5501) [ 247.076474][ T44] ? __might_fault (mm/memory.c:6716 mm/memory.c:6709) [ 247.076607][ T44] ? __pfx___lock_release (kernel/locking/lockdep.c:5477) [ 247.076737][ T44] ? trace_lock_acquire (./include/trace/events/lock.h:24 (discriminator 52)) [ 247.076867][ T44] ? __might_fault (mm/memory.c:6716 mm/memory.c:6709) [ 247.077002][ T44] ? lock_acquire (kernel/locking/lockdep.c:5798) [ 247.077142][ T44] ? fdget (./include/linux/atomic/atomic-arch-fallback.h:479 ./include/linux/atomic/atomic-instrumented.h:50 fs/file.c:1114 fs/file.c:1128) [ 247.077261][ T44] __sys_sendmsg (./include/linux/file.h:35 net/socket.c:2694) [ 247.077412][ T44] ? __pfx___sys_sendmsg (net/socket.c:2680) [ 247.077548][ T44] ? rseq_update_cpu_node_id (kernel/rseq.c:109 (discriminator 10)) [ 247.077687][ T44] ? __pfx___rseq_handle_notify_resume (kernel/rseq.c:316) [ 247.077853][ T44] do_syscall_64 (arch/x86/entry/common.c:52 arch/x86/entry/common.c:83) [ 247.077995][ T44] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:130) [ 247.078178][ T44] RIP: 0033:0x7f217acca7b7 [ 247.078324][ T44] RSP: 002b:00007ffd31957c18 EFLAGS: 00000246 ORIG_RAX: 000000000000002e [ 247.078543][ T44] RAX: ffffffffffffffda RBX: 00007ffd31958340 RCX: 00007f217acca7b7 [ 247.078733][ T44] RDX: 0000000000000000 RSI: 00007ffd31957c80 RDI: 0000000000000005 [ 247.078928][ T44] RBP: 0000000000000005 R08: 000000000000006c R09: 00007ffd319569f0 [ 247.079128][ T44] R10: 00007f217ab83708 R11: 0000000000000246 R12: 00007ffd31958368 Finger prints: __schedule:schedule:schedule_preempt_disabled:__mutex_lock:rtnl_newlink print_unlock_imbalance_bug:__lock_release:lock_release:__mutex_unlock_slowpath:__rtnl_unlock