====================================== | [ 42.015064][ T290] other info that might help us debug this: | [ 42.015278][ T290] no locks held by ip/290. | [ 42.015429][ T290] | [ 42.015429][ T290] stack backtrace: [ 42.015823][ T290] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 42.016139][ T290] Call Trace: [ 42.016254][ T290] [ 42.016331][ T290] dump_stack_lvl (lib/dump_stack.c:123) [ 42.016484][ T290] ? __rtnl_unlock (net/core/rtnetlink.c:140 (discriminator 3)) [ 42.016626][ T290] print_unlock_imbalance_bug (kernel/locking/lockdep.c:5266) [ 42.016771][ T290] __lock_release (kernel/locking/lockdep.c:5502) [ 42.016913][ T290] ? __pfx___lock_release (kernel/locking/lockdep.c:5477) [ 42.017061][ T290] ? __rtnl_unlock (net/core/rtnetlink.c:140 (discriminator 3)) [ 42.017205][ T290] lock_release (kernel/locking/lockdep.c:467 kernel/locking/lockdep.c:5848) [ 42.017354][ T290] __mutex_unlock_slowpath (./include/linux/instrumented.h:68 ./include/linux/atomic/atomic-instrumented.h:3188 kernel/locking/mutex.c:921) [ 42.017504][ T290] ? __pfx_lock_acquire.part.0 (kernel/locking/lockdep.c:5790) [ 42.017647][ T290] ? __pfx___mutex_unlock_slowpath (kernel/locking/mutex.c:907) [ 42.017825][ T290] ? rtnl_link_ops_get (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/core/rtnetlink.c:558) [ 42.017968][ T290] __rtnl_unlock (net/core/rtnetlink.c:140 (discriminator 3)) [ 42.018110][ T290] rtnl_newlink (net/core/rtnetlink.c:3961) [ 42.018252][ T290] ? __pfx_rtnl_newlink (net/core/rtnetlink.c:3919) [ 42.018392][ T290] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 42.018537][ T290] ? __lock_release (kernel/locking/lockdep.c:5501) [ 42.018681][ T290] ? rtnetlink_rcv_msg (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/core/rtnetlink.c:6926) [ 42.018822][ T290] ? __pfx___lock_release (kernel/locking/lockdep.c:5477) [ 42.018989][ T290] ? rtnetlink_rcv_msg (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/core/rtnetlink.c:6926) [ 42.019132][ T290] ? __pfx_rtnl_newlink (net/core/rtnetlink.c:3919) [ 42.019276][ T290] rtnetlink_rcv_msg (net/core/rtnetlink.c:6928) [ 42.019419][ T290] ? __pfx_rtnetlink_rcv_msg (net/core/rtnetlink.c:6831) [ 42.019561][ T290] ? hlock_class (./arch/x86/include/asm/bitops.h:227 ./arch/x86/include/asm/bitops.h:239 ./include/asm-generic/bitops/instrumented-non-atomic.h:142 kernel/locking/lockdep.c:228) [ 42.019703][ T290] ? mark_lock (kernel/locking/lockdep.c:4703 (discriminator 3)) [ 42.019815][ T290] ? __lock_acquire (kernel/locking/lockdep.c:5202) [ 42.019964][ T290] netlink_rcv_skb (net/netlink/af_netlink.c:2536) [ 42.020108][ T290] ? __pfx_rtnetlink_rcv_msg (net/core/rtnetlink.c:6831) [ 42.020249][ T290] ? __pfx_netlink_rcv_skb (net/netlink/af_netlink.c:2513) [ 42.020391][ T290] ? netlink_deliver_tap (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/netlink/af_netlink.c:340) [ 42.020535][ T290] ? netlink_deliver_tap (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 ./include/net/netns/generic.h:48 net/netlink/af_netlink.c:333) [ 42.020678][ T290] netlink_unicast (net/netlink/af_netlink.c:1316 net/netlink/af_netlink.c:1342) [ 42.020818][ T290] ? __pfx_netlink_unicast (net/netlink/af_netlink.c:1327) [ 42.020962][ T290] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 42.021109][ T290] netlink_sendmsg (net/netlink/af_netlink.c:1886) [ 42.021249][ T290] ? __pfx_netlink_sendmsg (net/netlink/af_netlink.c:1805) [ 42.021391][ T290] ____sys_sendmsg (net/socket.c:729 net/socket.c:744 net/socket.c:2609) [ 42.021535][ T290] ? __pfx_____sys_sendmsg (net/socket.c:2555) [ 42.021675][ T290] ? __pfx_copy_msghdr_from_user (net/socket.c:2535) [ 42.021859][ T290] ? hlock_class (./arch/x86/include/asm/bitops.h:227 ./arch/x86/include/asm/bitops.h:239 ./include/asm-generic/bitops/instrumented-non-atomic.h:142 kernel/locking/lockdep.c:228) [ 42.022003][ T290] ? validate_chain (./include/linux/hash.h:78 kernel/locking/lockdep.c:3794 kernel/locking/lockdep.c:3817 kernel/locking/lockdep.c:3872) [ 42.022150][ T290] ___sys_sendmsg (net/socket.c:2665) [ 42.022299][ T290] ? __pfx_validate_chain (kernel/locking/lockdep.c:3860) [ 42.022449][ T290] ? __pfx____sys_sendmsg (net/socket.c:2652) [ 42.022592][ T290] ? __lock_acquire (kernel/locking/lockdep.c:5202) [ 42.022736][ T290] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 42.022880][ T290] ? __lock_release (kernel/locking/lockdep.c:5501) [ 42.023024][ T290] ? __might_fault (mm/memory.c:6716 mm/memory.c:6709) [ 42.023168][ T290] ? fdget (./include/linux/atomic/atomic-arch-fallback.h:479 ./include/linux/atomic/atomic-instrumented.h:50 fs/file.c:1114 fs/file.c:1128) [ 42.023285][ T290] ? lock_acquire (kernel/locking/lockdep.c:5798) [ 42.023431][ T290] __sys_sendmsg (./include/linux/file.h:35 net/socket.c:2694) [ 42.023573][ T290] ? __pfx___sys_sendmsg (net/socket.c:2680) [ 42.023714][ T290] ? rseq_update_cpu_node_id (kernel/rseq.c:109 (discriminator 10)) [ 42.023868][ T290] do_syscall_64 (arch/x86/entry/common.c:52 arch/x86/entry/common.c:83) [ 42.024010][ T290] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:130) [ 42.024200][ T290] RIP: 0033:0x7fd1b37d47b7 [ 42.024347][ T290] Code: 0a 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b9 0f 1f 00 f3 0f 1e fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 2e 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 51 c3 48 83 ec 28 89 54 24 1c 48 89 74 24 10 All code ======== 0: 0a 00 or (%rax),%al 2: f7 d8 neg %eax 4: 64 89 02 mov %eax,%fs:(%rdx) 7: 48 c7 c0 ff ff ff ff mov $0xffffffffffffffff,%rax e: eb b9 jmp 0xffffffffffffffc9 10: 0f 1f 00 nopl (%rax) 13: f3 0f 1e fa endbr64 17: 64 8b 04 25 18 00 00 mov %fs:0x18,%eax 1e: 00 1f: 85 c0 test %eax,%eax 21: 75 10 jne 0x33 23: b8 2e 00 00 00 mov $0x2e,%eax 28: 0f 05 syscall 2a:* 48 3d 00 f0 ff ff cmp $0xfffffffffffff000,%rax <-- trapping instruction 30: 77 51 ja 0x83 32: c3 ret 33: 48 83 ec 28 sub $0x28,%rsp 37: 89 54 24 1c mov %edx,0x1c(%rsp) 3b: 48 89 74 24 10 mov %rsi,0x10(%rsp) Code starting with the faulting instruction =========================================== 0: 48 3d 00 f0 ff ff cmp $0xfffffffffffff000,%rax 6: 77 51 ja 0x59 8: c3 ret 9: 48 83 ec 28 sub $0x28,%rsp d: 89 54 24 1c mov %edx,0x1c(%rsp) 11: 48 89 74 24 10 mov %rsi,0x10(%rsp) [ 42.024846][ T290] RSP: 002b:00007ffe024bd148 EFLAGS: 00000246 ORIG_RAX: 000000000000002e [ 42.025068][ T290] RAX: ffffffffffffffda RBX: 00007ffe024bd880 RCX: 00007fd1b37d47b7 [ 42.025277][ T290] RDX: 0000000000000000 RSI: 00007ffe024bd1b0 RDI: 0000000000000005 [ 42.025500][ T290] RBP: 0000000000000002 R08: 0000000000000040 R09: 0000000000000000 [ 42.025713][ T290] R10: 00007fd1b3823ac0 R11: 0000000000000246 R12: 00007ffe024bd898 [ 42.025930][ T290] R13: 000000006729e186 R14: 0000000000496600 R15: 00007ffe024bd294 | WAIT TIMEOUT stderr | | Ctrl-C stderr | | Ctrl-C stderr [ 247.141086][ T43] INFO: task ip:290 blocked for more than 122 seconds. [ 247.141318][ T43] Not tainted 6.12.0-rc5-virtme #1 [ 247.141430][ T43] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 247.141621][ T43] task:ip state:D stack:25816 pid:290 tgid:290 ppid:237 flags:0x00004006 [ 247.141853][ T43] Call Trace: [ 247.141948][ T43] [ 247.142038][ T43] __schedule (kernel/sched/core.c:5328 kernel/sched/core.c:6690) [ 247.142215][ T43] ? __pfx___schedule (kernel/sched/core.c:6546) [ 247.142333][ T43] ? trace_lock_acquire (./include/trace/events/lock.h:24 (discriminator 52)) [ 247.142453][ T43] ? trace_lock_release (./include/trace/events/lock.h:69 (discriminator 52)) [ 247.142570][ T43] schedule (./arch/x86/include/asm/preempt.h:84 kernel/sched/core.c:6768 kernel/sched/core.c:6782) [ 247.142656][ T43] schedule_preempt_disabled (./arch/x86/include/asm/preempt.h:79 kernel/sched/core.c:6840) [ 247.142771][ T43] __mutex_lock (kernel/locking/mutex.c:197 kernel/locking/mutex.c:686 kernel/locking/mutex.c:752) [ 247.142887][ T43] ? rtnl_newlink (net/core/rtnetlink.c:321 net/core/rtnetlink.c:4025) [ 247.143017][ T43] ? __pfx___mutex_lock (kernel/locking/mutex.c:751) [ 247.143129][ T43] ? trace_lock_acquire (./include/trace/events/lock.h:24 (discriminator 52)) [ 247.143254][ T43] ? rtnl_newlink (net/core/rtnetlink.c:321 net/core/rtnetlink.c:4025) [ 247.143371][ T43] rtnl_newlink (net/core/rtnetlink.c:321 net/core/rtnetlink.c:4025) [ 247.143488][ T43] ? __pfx_rtnl_newlink (net/core/rtnetlink.c:3919) [ 247.143603][ T43] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 247.143720][ T43] ? __pfx___lock_release (kernel/locking/lockdep.c:5477) [ 247.143847][ T43] ? rtnetlink_rcv_msg (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/core/rtnetlink.c:6926) [ 247.143976][ T43] ? __pfx_rtnl_newlink (net/core/rtnetlink.c:3919) [ 247.144106][ T43] rtnetlink_rcv_msg (net/core/rtnetlink.c:6928) [ 247.144222][ T43] ? __pfx_rtnetlink_rcv_msg (net/core/rtnetlink.c:6831) [ 247.144337][ T43] ? hlock_class (./arch/x86/include/asm/bitops.h:227 ./arch/x86/include/asm/bitops.h:239 ./include/asm-generic/bitops/instrumented-non-atomic.h:142 kernel/locking/lockdep.c:228) [ 247.144490][ T43] ? mark_lock (kernel/locking/lockdep.c:4703 (discriminator 3)) [ 247.144605][ T43] ? __lock_acquire (kernel/locking/lockdep.c:5202) [ 247.144718][ T43] netlink_rcv_skb (net/netlink/af_netlink.c:2536) [ 247.144831][ T43] ? __pfx_rtnetlink_rcv_msg (net/core/rtnetlink.c:6831) [ 247.144948][ T43] ? __pfx_netlink_rcv_skb (net/netlink/af_netlink.c:2513) [ 247.145079][ T43] ? netlink_deliver_tap (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 net/netlink/af_netlink.c:340) [ 247.145197][ T43] ? netlink_deliver_tap (./include/linux/rcupdate.h:347 ./include/linux/rcupdate.h:880 ./include/net/netns/generic.h:48 net/netlink/af_netlink.c:333) [ 247.145307][ T43] netlink_unicast (net/netlink/af_netlink.c:1316 net/netlink/af_netlink.c:1342) [ 247.145418][ T43] ? __pfx_netlink_unicast (net/netlink/af_netlink.c:1327) [ 247.145531][ T43] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 247.145644][ T43] netlink_sendmsg (net/netlink/af_netlink.c:1886) [ 247.145757][ T43] ? __pfx_netlink_sendmsg (net/netlink/af_netlink.c:1805) [ 247.145874][ T43] ____sys_sendmsg (net/socket.c:729 net/socket.c:744 net/socket.c:2609) [ 247.146006][ T43] ? __pfx_____sys_sendmsg (net/socket.c:2555) [ 247.146123][ T43] ? __pfx_copy_msghdr_from_user (net/socket.c:2535) [ 247.146265][ T43] ? hlock_class (./arch/x86/include/asm/bitops.h:227 ./arch/x86/include/asm/bitops.h:239 ./include/asm-generic/bitops/instrumented-non-atomic.h:142 kernel/locking/lockdep.c:228) [ 247.146380][ T43] ? validate_chain (./include/linux/hash.h:78 kernel/locking/lockdep.c:3794 kernel/locking/lockdep.c:3817 kernel/locking/lockdep.c:3872) [ 247.146491][ T43] ___sys_sendmsg (net/socket.c:2665) [ 247.146605][ T43] ? __pfx_validate_chain (kernel/locking/lockdep.c:3860) [ 247.146715][ T43] ? __pfx____sys_sendmsg (net/socket.c:2652) [ 247.146829][ T43] ? __lock_acquire (kernel/locking/lockdep.c:5202) [ 247.146954][ T43] ? find_held_lock (kernel/locking/lockdep.c:5315) [ 247.147081][ T43] ? __lock_release (kernel/locking/lockdep.c:5501) [ 247.147192][ T43] ? __might_fault (mm/memory.c:6716 mm/memory.c:6709) [ 247.147314][ T43] ? fdget (./include/linux/atomic/atomic-arch-fallback.h:479 ./include/linux/atomic/atomic-instrumented.h:50 fs/file.c:1114 fs/file.c:1128) [ 247.147401][ T43] ? lock_acquire (kernel/locking/lockdep.c:5798) [ 247.147515][ T43] __sys_sendmsg (./include/linux/file.h:35 net/socket.c:2694) [ 247.147632][ T43] ? __pfx___sys_sendmsg (net/socket.c:2680) [ 247.147743][ T43] ? rseq_update_cpu_node_id (kernel/rseq.c:109 (discriminator 10)) [ 247.147860][ T43] do_syscall_64 (arch/x86/entry/common.c:52 arch/x86/entry/common.c:83) [ 247.147985][ T43] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:130) [ 247.148138][ T43] RIP: 0033:0x7fd1b37d47b7 [ 247.148257][ T43] RSP: 002b:00007ffe024bd148 EFLAGS: 00000246 ORIG_RAX: 000000000000002e [ 247.148447][ T43] RAX: ffffffffffffffda RBX: 00007ffe024bd880 RCX: 00007fd1b37d47b7 [ 247.148632][ T43] RDX: 0000000000000000 RSI: 00007ffe024bd1b0 RDI: 0000000000000005 [ 247.148816][ T43] RBP: 0000000000000002 R08: 0000000000000040 R09: 0000000000000000 [ 247.149015][ T43] R10: 00007fd1b3823ac0 R11: 0000000000000246 R12: 00007ffe024bd898 Finger prints: __schedule:schedule:schedule_preempt_disabled:__mutex_lock:rtnl_newlink print_unlock_imbalance_bug:__lock_release:lock_release:__mutex_unlock_slowpath:__rtnl_unlock