====================================== | [ 40.332448][ T292] other info that might help us debug this: | [ 40.332730][ T292] no locks held by ip/292. | [ 40.332934][ T292] | [ 40.332934][ T292] stack backtrace: [ 40.333480][ T292] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 40.333944][ T292] Call Trace: [ 40.334106][ T292] [ 40.334217][ T292] dump_stack_lvl (lib/dump_stack.c:123) [ 40.334428][ T292] ? __rtnl_unlock (net/core/rtnetlink.c:140 (discriminator 3)) [ 40.334637][ T292] print_unlock_imbalance_bug (kernel/locking/lockdep.c:5266) [ 40.334838][ T292] __lock_release (kernel/locking/lockdep.c:5502) [ 40.335045][ T292] ? __pfx___lock_release (kernel/locking/lockdep.c:5477) [ 40.335258][ T292] ? __rtnl_unlock (net/core/rtnetlink.c:140 (discriminator 3)) [ 40.335457][ T292] lock_release (kernel/locking/lockdep.c:467 kernel/locking/lockdep.c:5848) [ 40.335670][ T292] __mutex_unlock_slowpath (./include/linux/instrumented.h:68 ./include/linux/atomic/atomic-instrumented.h:3188 kernel/locking/mutex.c:921) [ 40.335876][ T292] ? __pfx_lock_acquire.part.0 (kernel/locking/lockdep.c:5790) [ 40.336081][ T292] ? __pfx___mutex_unlock_slowpath (kernel/locking/mutex.c:907) [ 40.336326][ T292] ? rtnl_link_ops_get (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/core/rtnetlink.c:558) [ 40.336530][ T292] __rtnl_unlock (net/core/rtnetlink.c:140 (discriminator 3)) [ 40.336734][ T292] rtnl_newlink (net/core/rtnetlink.c:3961) [ 40.336930][ T292] ? __pfx_rtnl_newlink (net/core/rtnetlink.c:3919) [ 40.337137][ T292] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 40.337350][ T292] ? __lock_release (kernel/locking/lockdep.c:5501) [ 40.337544][ T292] ? rtnetlink_rcv_msg (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/core/rtnetlink.c:6926) [ 40.337764][ T292] ? __pfx___lock_release (kernel/locking/lockdep.c:5477) [ 40.337972][ T292] ? rtnetlink_rcv_msg (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/core/rtnetlink.c:6926) [ 40.338178][ T292] ? __pfx_rtnl_newlink (net/core/rtnetlink.c:3919) [ 40.338419][ T292] rtnetlink_rcv_msg (net/core/rtnetlink.c:6928) [ 40.338628][ T292] ? __pfx_rtnetlink_rcv_msg (net/core/rtnetlink.c:6831) [ 40.338836][ T292] ? hlock_class (./arch/x86/include/asm/bitops.h:227 ./arch/x86/include/asm/bitops.h:239 ./include/asm-generic/bitops/instrumented-non-atomic.h:142 kernel/locking/lockdep.c:228) [ 40.339043][ T292] ? mark_lock (kernel/locking/lockdep.c:4703 (discriminator 3)) [ 40.339219][ T292] ? __lock_acquire (kernel/locking/lockdep.c:5202) [ 40.339423][ T292] netlink_rcv_skb (net/netlink/af_netlink.c:2536) [ 40.339625][ T292] ? __pfx_rtnetlink_rcv_msg (net/core/rtnetlink.c:6831) [ 40.339830][ T292] ? __pfx_netlink_rcv_skb (net/netlink/af_netlink.c:2513) [ 40.340041][ T292] ? netlink_deliver_tap (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/netlink/af_netlink.c:340) [ 40.340244][ T292] ? netlink_deliver_tap (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 ./include/net/netns/generic.h:48 net/netlink/af_netlink.c:333) [ 40.340450][ T292] netlink_unicast (net/netlink/af_netlink.c:1316 net/netlink/af_netlink.c:1342) [ 40.340667][ T292] ? __pfx_netlink_unicast (net/netlink/af_netlink.c:1327) [ 40.340866][ T292] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 40.341077][ T292] netlink_sendmsg (net/netlink/af_netlink.c:1886) [ 40.341285][ T292] ? __pfx_netlink_sendmsg (net/netlink/af_netlink.c:1805) [ 40.341486][ T292] ____sys_sendmsg (net/socket.c:729 net/socket.c:744 net/socket.c:2609) [ 40.341690][ T292] ? __pfx_____sys_sendmsg (net/socket.c:2555) [ 40.341897][ T292] ? __pfx_copy_msghdr_from_user (net/socket.c:2535) [ 40.342139][ T292] ? hlock_class (./arch/x86/include/asm/bitops.h:227 ./arch/x86/include/asm/bitops.h:239 ./include/asm-generic/bitops/instrumented-non-atomic.h:142 kernel/locking/lockdep.c:228) [ 40.342351][ T292] ? validate_chain (./include/linux/hash.h:78 kernel/locking/lockdep.c:3794 kernel/locking/lockdep.c:3817 kernel/locking/lockdep.c:3872) [ 40.342557][ T292] ___sys_sendmsg (net/socket.c:2665) [ 40.342768][ T292] ? __pfx_validate_chain (kernel/locking/lockdep.c:3860) [ 40.342967][ T292] ? __pfx____sys_sendmsg (net/socket.c:2652) [ 40.343165][ T292] ? __lock_acquire (kernel/locking/lockdep.c:5202) [ 40.343373][ T292] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 40.343579][ T292] ? __lock_release (kernel/locking/lockdep.c:5501) [ 40.343778][ T292] ? __might_fault (mm/memory.c:6716 mm/memory.c:6709) [ 40.343997][ T292] ? fdget (./include/linux/atomic/atomic-arch-fallback.h:479 ./include/linux/atomic/atomic-instrumented.h:50 fs/file.c:1114 fs/file.c:1128) [ 40.344151][ T292] ? lock_acquire (kernel/locking/lockdep.c:5798) [ 40.344355][ T292] __sys_sendmsg (./include/linux/file.h:35 net/socket.c:2694) [ 40.344556][ T292] ? __pfx___sys_sendmsg (net/socket.c:2680) [ 40.344792][ T292] ? rseq_update_cpu_node_id (kernel/rseq.c:109 (discriminator 10)) [ 40.345012][ T292] do_syscall_64 (arch/x86/entry/common.c:52 arch/x86/entry/common.c:83) [ 40.345225][ T292] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:130) [ 40.345487][ T292] RIP: 0033:0x7faff40007b7 [ 40.345697][ T292] Code: 0a 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b9 0f 1f 00 f3 0f 1e fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 2e 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 51 c3 48 83 ec 28 89 54 24 1c 48 89 74 24 10 All code ======== 0: 0a 00 or (%rax),%al 2: f7 d8 neg %eax 4: 64 89 02 mov %eax,%fs:(%rdx) 7: 48 c7 c0 ff ff ff ff mov $0xffffffffffffffff,%rax e: eb b9 jmp 0xffffffffffffffc9 10: 0f 1f 00 nopl (%rax) 13: f3 0f 1e fa endbr64 17: 64 8b 04 25 18 00 00 mov %fs:0x18,%eax 1e: 00 1f: 85 c0 test %eax,%eax 21: 75 10 jne 0x33 23: b8 2e 00 00 00 mov $0x2e,%eax 28: 0f 05 syscall 2a:* 48 3d 00 f0 ff ff cmp $0xfffffffffffff000,%rax <-- trapping instruction 30: 77 51 ja 0x83 32: c3 ret 33: 48 83 ec 28 sub $0x28,%rsp 37: 89 54 24 1c mov %edx,0x1c(%rsp) 3b: 48 89 74 24 10 mov %rsi,0x10(%rsp) Code starting with the faulting instruction =========================================== 0: 48 3d 00 f0 ff ff cmp $0xfffffffffffff000,%rax 6: 77 51 ja 0x59 8: c3 ret 9: 48 83 ec 28 sub $0x28,%rsp d: 89 54 24 1c mov %edx,0x1c(%rsp) 11: 48 89 74 24 10 mov %rsi,0x10(%rsp) [ 40.346424][ T292] RSP: 002b:00007ffe305b7e78 EFLAGS: 00000246 ORIG_RAX: 000000000000002e [ 40.346747][ T292] RAX: ffffffffffffffda RBX: 00007ffe305b85b0 RCX: 00007faff40007b7 [ 40.347058][ T292] RDX: 0000000000000000 RSI: 00007ffe305b7ee0 RDI: 0000000000000005 [ 40.347374][ T292] RBP: 0000000000000002 R08: 0000000000000040 R09: 0000000000000000 [ 40.347695][ T292] R10: 00007faff404fac0 R11: 0000000000000246 R12: 00007ffe305b85c8 [ 40.347894][ T292] R13: 00000000672a0b9b R14: 0000000000496600 R15: 00007ffe305b7fc4 | WAIT TIMEOUT stderr | | Ctrl-C stderr | | Ctrl-C stderr [ 247.118252][ T43] INFO: task ip:292 blocked for more than 122 seconds. [ 247.118579][ T43] Not tainted 6.12.0-rc5-virtme #1 [ 247.118706][ T43] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 247.118918][ T43] task:ip state:D stack:26008 pid:292 tgid:292 ppid:238 flags:0x00004006 [ 247.119195][ T43] Call Trace: [ 247.119294][ T43] [ 247.119374][ T43] __schedule (kernel/sched/core.c:5328 kernel/sched/core.c:6690) [ 247.119512][ T43] ? __pfx___schedule (kernel/sched/core.c:6546) [ 247.119645][ T43] ? trace_lock_acquire (./include/trace/events/lock.h:24 (discriminator 52)) [ 247.119780][ T43] ? trace_lock_release (./include/trace/events/lock.h:69 (discriminator 52)) [ 247.119906][ T43] schedule (./arch/x86/include/asm/preempt.h:84 kernel/sched/core.c:6768 kernel/sched/core.c:6782) [ 247.120007][ T43] schedule_preempt_disabled (./arch/x86/include/asm/preempt.h:79 kernel/sched/core.c:6840) [ 247.120136][ T43] __mutex_lock (kernel/locking/mutex.c:197 kernel/locking/mutex.c:686 kernel/locking/mutex.c:752) [ 247.120275][ T43] ? rtnl_newlink (net/core/rtnetlink.c:321 net/core/rtnetlink.c:4025) [ 247.120417][ T43] ? __pfx___mutex_lock (kernel/locking/mutex.c:751) [ 247.120542][ T43] ? trace_lock_acquire (./include/trace/events/lock.h:24 (discriminator 52)) [ 247.120671][ T43] ? rtnl_newlink (net/core/rtnetlink.c:321 net/core/rtnetlink.c:4025) [ 247.120795][ T43] rtnl_newlink (net/core/rtnetlink.c:321 net/core/rtnetlink.c:4025) [ 247.120923][ T43] ? __pfx_rtnl_newlink (net/core/rtnetlink.c:3919) [ 247.121047][ T43] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 247.121180][ T43] ? __pfx___lock_release (kernel/locking/lockdep.c:5477) [ 247.121310][ T43] ? rtnetlink_rcv_msg (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/core/rtnetlink.c:6926) [ 247.121438][ T43] ? __pfx_rtnl_newlink (net/core/rtnetlink.c:3919) [ 247.121567][ T43] rtnetlink_rcv_msg (net/core/rtnetlink.c:6928) [ 247.121694][ T43] ? __pfx_rtnetlink_rcv_msg (net/core/rtnetlink.c:6831) [ 247.121824][ T43] ? hlock_class (./arch/x86/include/asm/bitops.h:227 ./arch/x86/include/asm/bitops.h:239 ./include/asm-generic/bitops/instrumented-non-atomic.h:142 kernel/locking/lockdep.c:228) [ 247.121950][ T43] ? mark_lock (kernel/locking/lockdep.c:4703 (discriminator 3)) [ 247.122048][ T43] ? __lock_acquire (kernel/locking/lockdep.c:5202) [ 247.122182][ T43] netlink_rcv_skb (net/netlink/af_netlink.c:2536) [ 247.122310][ T43] ? __pfx_rtnetlink_rcv_msg (net/core/rtnetlink.c:6831) [ 247.122435][ T43] ? __pfx_netlink_rcv_skb (net/netlink/af_netlink.c:2513) [ 247.122569][ T43] ? netlink_deliver_tap (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/netlink/af_netlink.c:340) [ 247.122696][ T43] ? netlink_deliver_tap (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 ./include/net/netns/generic.h:48 net/netlink/af_netlink.c:333) [ 247.122819][ T43] netlink_unicast (net/netlink/af_netlink.c:1316 net/netlink/af_netlink.c:1342) [ 247.122944][ T43] ? __pfx_netlink_unicast (net/netlink/af_netlink.c:1327) [ 247.123067][ T43] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 247.123201][ T43] netlink_sendmsg (net/netlink/af_netlink.c:1886) [ 247.123323][ T43] ? __pfx_netlink_sendmsg (net/netlink/af_netlink.c:1805) [ 247.123448][ T43] ____sys_sendmsg (net/socket.c:729 net/socket.c:744 net/socket.c:2609) [ 247.123579][ T43] ? __pfx_____sys_sendmsg (net/socket.c:2555) [ 247.123703][ T43] ? __pfx_copy_msghdr_from_user (net/socket.c:2535) [ 247.123854][ T43] ? hlock_class (./arch/x86/include/asm/bitops.h:227 ./arch/x86/include/asm/bitops.h:239 ./include/asm-generic/bitops/instrumented-non-atomic.h:142 kernel/locking/lockdep.c:228) [ 247.123980][ T43] ? validate_chain (./include/linux/hash.h:78 kernel/locking/lockdep.c:3794 kernel/locking/lockdep.c:3817 kernel/locking/lockdep.c:3872) [ 247.124105][ T43] ___sys_sendmsg (net/socket.c:2665) [ 247.124237][ T43] ? __pfx_validate_chain (kernel/locking/lockdep.c:3860) [ 247.124361][ T43] ? __pfx____sys_sendmsg (net/socket.c:2652) [ 247.124487][ T43] ? __lock_acquire (kernel/locking/lockdep.c:5202) [ 247.124622][ T43] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 247.124752][ T43] ? __lock_release (kernel/locking/lockdep.c:5501) [ 247.124877][ T43] ? __might_fault (mm/memory.c:6716 mm/memory.c:6709) [ 247.125003][ T43] ? fdget (./include/linux/atomic/atomic-arch-fallback.h:479 ./include/linux/atomic/atomic-instrumented.h:50 fs/file.c:1114 fs/file.c:1128) [ 247.125101][ T43] ? lock_acquire (kernel/locking/lockdep.c:5798) [ 247.125234][ T43] __sys_sendmsg (./include/linux/file.h:35 net/socket.c:2694) [ 247.125357][ T43] ? __pfx___sys_sendmsg (net/socket.c:2680) [ 247.125482][ T43] ? rseq_update_cpu_node_id (kernel/rseq.c:109 (discriminator 10)) [ 247.125615][ T43] do_syscall_64 (arch/x86/entry/common.c:52 arch/x86/entry/common.c:83) [ 247.125744][ T43] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:130) [ 247.125904][ T43] RIP: 0033:0x7faff40007b7 [ 247.126041][ T43] RSP: 002b:00007ffe305b7e78 EFLAGS: 00000246 ORIG_RAX: 000000000000002e [ 247.126250][ T43] RAX: ffffffffffffffda RBX: 00007ffe305b85b0 RCX: 00007faff40007b7 [ 247.126442][ T43] RDX: 0000000000000000 RSI: 00007ffe305b7ee0 RDI: 0000000000000005 [ 247.126624][ T43] RBP: 0000000000000002 R08: 0000000000000040 R09: 0000000000000000 [ 247.126813][ T43] R10: 00007faff404fac0 R11: 0000000000000246 R12: 00007ffe305b85c8 Finger prints: __schedule:schedule:schedule_preempt_disabled:__mutex_lock:rtnl_newlink print_unlock_imbalance_bug:__lock_release:lock_release:__mutex_unlock_slowpath:__rtnl_unlock