======================================
| [  349.672992][ T3527] netdevsim netdevsim692 eni692np1: renamed from eth1
| [  353.026861][    C3] ------------[ cut here ]------------
| [  353.027493][    C3] WARNING: CPU: 3 PID: 3567 at ./include/net/sch_generic.h:843 __dev_xmit_skb (./include/net/sch_generic.h:843 ./include/net/sch_generic.h:889 ./include/net/sch_generic.h:895 net/core/dev.c:4168)
| [  353.028279][    C3] Modules linked in: netdevsim psample pktgen vxlan xfrm_user vxcan can_dev xfrm_interface ip6_gre ip_gre gre macsec ipvlan
[  353.029975][    C3] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011
[  353.030428][    C3] RIP: 0010:__dev_xmit_skb (./include/net/sch_generic.h:843 ./include/net/sch_generic.h:889 ./include/net/sch_generic.h:895 net/core/dev.c:4168)
[  353.030801][    C3] Code: ea 03 0f b6 04 02 48 89 fa 83 e2 07 38 d0 7f 09 84 c0 74 05 e8 1c c6 e8 fe 41 c6 87 b0 01 00 00 00 45 31 ed e9 06 f6 ff ff 90 <0f> 0b 90 e9 00 ff ff ff be 08 00 00 00 48 89 ef e8 56 cc e8 fe f0
All code
========
   0:	ea                   	(bad)
   1:	03 0f                	add    (%rdi),%ecx
   3:	b6 04                	mov    $0x4,%dh
   5:	02 48 89             	add    -0x77(%rax),%cl
   8:	fa                   	cli
   9:	83 e2 07             	and    $0x7,%edx
   c:	38 d0                	cmp    %dl,%al
   e:	7f 09                	jg     0x19
  10:	84 c0                	test   %al,%al
  12:	74 05                	je     0x19
  14:	e8 1c c6 e8 fe       	call   0xfffffffffee8c635
  19:	41 c6 87 b0 01 00 00 	movb   $0x0,0x1b0(%r15)
  20:	00 
  21:	45 31 ed             	xor    %r13d,%r13d
  24:	e9 06 f6 ff ff       	jmp    0xfffffffffffff62f
  29:	90                   	nop
  2a:*	0f 0b                	ud2		<-- trapping instruction
  2c:	90                   	nop
  2d:	e9 00 ff ff ff       	jmp    0xffffffffffffff32
  32:	be 08 00 00 00       	mov    $0x8,%esi
  37:	48 89 ef             	mov    %rbp,%rdi
  3a:	e8 56 cc e8 fe       	call   0xfffffffffee8cc95
  3f:	f0                   	lock

Code starting with the faulting instruction
===========================================
   0:	0f 0b                	ud2
   2:	90                   	nop
   3:	e9 00 ff ff ff       	jmp    0xffffffffffffff08
   8:	be 08 00 00 00       	mov    $0x8,%esi
   d:	48 89 ef             	mov    %rbp,%rdi
  10:	e8 56 cc e8 fe       	call   0xfffffffffee8cc6b
  15:	f0                   	lock
[  353.032113][    C3] RSP: 0018:ffffc900002706a0 EFLAGS: 00010202
[  353.032578][    C3] RAX: 0000000000000001 RBX: ffffe8ffffd8d5a0 RCX: 0000000000000000
[  353.033149][    C3] RDX: 0000000000000007 RSI: ffffffff976602c0 RDI: ffff888018648886
[  353.034039][    C3] RBP: ffff88801585a198 R08: 0000000000000001 R09: 0000000000000001
[  353.034584][    C3] R10: 000000000000a000 R11: ffff88801585a228 R12: ffff88801cff0c28
[  353.035136][    C3] R13: ffff888018648880 R14: ffff88801585a210 R15: ffff88801585a000
[  353.035674][    C3] FS:  00007f518baaab80(0000) GS:ffff8880d3688000(0000) knlGS:0000000000000000
[  353.036340][    C3] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  353.036804][    C3] CR2: 00007f518bc25720 CR3: 000000001b8fc006 CR4: 0000000000772ef0
[  353.037384][    C3] PKRU: 55555554
[  353.037662][    C3] Call Trace:
[  353.037963][    C3]  <IRQ>
[  353.038153][    C3]  ? napi_hash_add (net/core/dev.c:4073)
[  353.038525][    C3]  ? dev_get_stats (net/core/dev.c:4143)
[  353.038900][    C3]  __dev_queue_xmit (net/core/dev.c:4786)
[  353.039296][    C3]  ? __lock_acquire (kernel/locking/lockdep.c:5237)
[  353.039664][    C3]  ? netdev_core_pick_tx (net/core/dev.c:4728)
[  353.040048][    C3]  ? eth_header (net/ethernet/eth.c:100)
[  353.040413][    C3]  ? neigh_resolve_output (./include/linux/netdevice.h:3422 net/core/neighbour.c:1589 net/core/neighbour.c:1574)
[  353.040782][    C3]  ? __neigh_update (./include/linux/rcupdate.h:331 ./include/linux/rcupdate.h:867 net/core/neighbour.c:1476)
[  353.041152][    C3]  __neigh_update (./include/linux/instrumented.h:96 (discriminator 2) ./include/linux/atomic/atomic-instrumented.h:400 (discriminator 2) ./include/linux/refcount.h:389 (discriminator 2) ./include/linux/refcount.h:432 (discriminator 2) ./include/linux/refcount.h:450 (discriminator 2) ./include/net/neighbour.h:458 (discriminator 2) net/core/neighbour.c:1494 (discriminator 2))
[  353.041538][    C3]  arp_process (./include/linux/instrumented.h:96 (discriminator 4) ./include/linux/atomic/atomic-instrumented.h:400 (discriminator 4) ./include/linux/refcount.h:389 (discriminator 4) ./include/linux/refcount.h:432 (discriminator 4) ./include/linux/refcount.h:450 (discriminator 4) ./include/net/neighbour.h:458 (discriminator 4) net/ipv4/arp.c:937 (discriminator 4))
[  353.041936][    C3]  ? arp_send (net/ipv4/arp.c:702)
[  353.042206][    C3]  ? arch_stack_walk (arch/x86/kernel/stacktrace.c:24)
[  353.042574][    C3]  ? __lock_acquire (kernel/locking/lockdep.c:5237)
[  353.042962][    C3]  ? arp_rcv.part.0 (net/ipv4/arp.c:968)
[  353.043334][    C3]  __netif_receive_skb_one_core (net/core/dev.c:6131)
[  353.043780][    C3]  ? __netif_receive_skb_list_core (net/core/dev.c:6131)
[  353.044233][    C3]  ? rcu_is_watching (./include/linux/context_tracking.h:128 kernel/rcu/tree.c:751)
[  353.044600][    C3]  ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5831)
[  353.044983][    C3]  ? netif_receive_skb_internal (./include/linux/rcupdate.h:331 (discriminator 16) ./include/linux/rcupdate.h:867 (discriminator 16) net/core/dev.c:6324 (discriminator 16))
[  353.045460][    C3]  netif_receive_skb_internal (net/core/dev.c:6337)
[  353.045831][    C3]  ? __netif_receive_skb (net/core/dev.c:6316)
[  353.046210][    C3]  ? mark_held_locks (kernel/locking/lockdep.c:4325)
[  353.046580][    C3]  ? _raw_spin_unlock_irqrestore (./include/linux/spinlock_api_smp.h:151 kernel/locking/spinlock.c:194)
[  353.047065][    C3]  ? lockdep_hardirqs_on (kernel/locking/lockdep.c:4472)
[  353.047445][    C3]  ? rcu_is_watching (./include/linux/context_tracking.h:128 kernel/rcu/tree.c:751)
[  353.047840][    C3]  netif_receive_skb (net/core/dev.c:6396)
[  353.048223][    C3] nsim_poll (drivers/net/netdevsim/netdev.c:439 drivers/net/netdevsim/netdev.c:454) netdevsim
[  353.048634][    C3]  ? nsim_start_peer_tx_queue.isra.0 (drivers/net/netdevsim/netdev.c:450) netdevsim
[  353.049219][    C3]  ? rcu_is_watching (./include/linux/context_tracking.h:128 kernel/rcu/tree.c:751)
[  353.049604][    C3]  __napi_poll.constprop.0 (net/core/dev.c:7668)
[  353.049990][    C3]  net_rx_action (net/core/dev.c:7730 net/core/dev.c:7882)
[  353.050380][    C3]  ? run_backlog_napi (net/core/dev.c:7844)
[  353.050768][    C3]  ? clockevents_program_event (kernel/time/clockevents.c:326)
[  353.051162][    C3]  ? lock_downgrade (kernel/locking/lockdep.c:471 kernel/locking/lockdep.c:5764)
[  353.051550][    C3]  ? kvm_clock_get_cycles (./arch/x86/include/asm/preempt.h:95 arch/x86/kernel/kvmclock.c:80 arch/x86/kernel/kvmclock.c:86)
[  353.051933][    C3]  ? ktime_get (kernel/time/timekeeping.c:387 (discriminator 4) kernel/time/timekeeping.c:404 (discriminator 4) kernel/time/timekeeping.c:826 (discriminator 4))
[  353.052226][    C3]  ? clockevents_program_event (kernel/time/clockevents.c:336 (discriminator 3))
[  353.052702][    C3]  handle_softirqs (kernel/softirq.c:623)
[  353.053126][    C3]  ? __neigh_event_send (./include/linux/bottom_half.h:33 net/core/neighbour.c:1273)
[  353.053501][    C3]  do_softirq (kernel/softirq.c:523 kernel/softirq.c:510)
[  353.053780][    C3]  </IRQ>
[  353.053991][    C3]  <TASK>
[  353.054175][    C3]  __local_bh_enable_ip (kernel/softirq.c:450)
[  353.054532][    C3]  __neigh_event_send (net/core/neighbour.c:1274)
[  353.054936][    C3]  neigh_resolve_output (net/core/neighbour.c:1578)
[  353.055314][    C3]  ? __local_bh_enable_ip (./arch/x86/include/asm/irqflags.h:42 ./arch/x86/include/asm/irqflags.h:119 kernel/softirq.c:455)
[  353.055689][    C3]  ? ___neigh_create (net/core/neighbour.c:727)
[  353.056072][    C3]  ip_finish_output2 (./include/net/neighbour.h:556 net/ipv4/ip_output.c:237)
[  353.056440][    C3]  ? ip_skb_dst_mtu (./include/linux/rcupdate.h:341 ./include/linux/rcupdate.h:897 ./include/net/ip.h:504 ./include/net/ip.h:518)
[  353.056799][    C3]  ? ip_append_data.part.0 (net/ipv4/ip_output.c:201)
[  353.057166][    C3]  ? ip_skb_dst_mtu (./include/net/ip.h:523)
[  353.057539][    C3]  ? __ip_finish_output (./include/linux/skbuff.h:1724 ./include/linux/skbuff.h:5149 net/ipv4/ip_output.c:309 net/ipv4/ip_output.c:297)
[  353.057919][    C3]  ip_output (./include/linux/netfilter.h:307 net/ipv4/ip_output.c:438)
[  353.058491][    C3]  ? ip_finish_output (net/ipv4/ip_output.c:429)
[  353.058858][    C3]  ? find_held_lock (kernel/locking/lockdep.c:5350)
[  353.059241][    C3]  __ip_queue_xmit (./include/net/dst.h:464 net/ipv4/ip_output.c:131 net/ipv4/ip_output.c:534)
[  353.059620][    C3]  ? __skb_clone (./arch/x86/include/asm/atomic.h:53 (discriminator 4) ./include/linux/atomic/atomic-arch-fallback.h:992 (discriminator 4) ./include/linux/atomic/atomic-instrumented.h:436 (discriminator 4) net/core/skbuff.c:1585 (discriminator 4))
[  353.060003][    C3]  __tcp_transmit_skb (net/ipv4/tcp_output.c:1628)
[  353.060376][    C3]  ? __tcp_select_window (net/ipv4/tcp_output.c:1449)
[  353.060750][    C3]  ? tcp_call_bpf (./include/net/sock.h:1685 ./include/net/sock.h:1780 ./include/net/tcp.h:2775)
[  353.061140][    C3]  tcp_connect (net/ipv4/tcp_output.c:1646 net/ipv4/tcp_output.c:4335)
[  353.061785][    C3]  tcp_v4_connect (net/ipv4/tcp_ipv4.c:347)
[  353.062179][    C3]  ? tcp_v4_conn_request (net/ipv4/tcp_ipv4.c:225)
[  353.062536][    C3]  ? do_raw_spin_lock (./arch/x86/include/asm/atomic.h:107 ./include/linux/atomic/atomic-arch-fallback.h:2170 ./include/linux/atomic/atomic-instrumented.h:1302 ./include/asm-generic/qspinlock.h:111 kernel/locking/spinlock_debug.c:116)
[  353.062903][    C3]  ? find_held_lock (kernel/locking/lockdep.c:5350)
[  353.063560][    C3]  __inet_stream_connect (net/ipv4/af_inet.c:679)
[  353.063932][    C3]  ? __local_bh_enable_ip (./arch/x86/include/asm/irqflags.h:42 ./arch/x86/include/asm/irqflags.h:119 kernel/softirq.c:455)
[  353.064308][    C3]  ? __inet_stream_connect (net/ipv4/af_inet.c:746)
[  353.064686][    C3]  inet_stream_connect (net/ipv4/af_inet.c:751)
[  353.065314][    C3]  __sys_connect (./include/linux/file.h:62 ./include/linux/file.h:83 net/socket.c:2111)
[  353.065671][    C3]  ? __sys_connect_file (net/socket.c:2109)
[  353.066035][    C3]  ? do_fcntl (fs/fcntl.c:471)
[  353.066312][    C3]  ? f_getown (fs/fcntl.c:446)
[  353.066592][    C3]  ? fd_install (./arch/x86/include/asm/preempt.h:104 ./include/linux/rcupdate.h:981 fs/file.c:676)
[  353.067235][    C3]  ? __sys_socket (net/socket.c:516 net/socket.c:1756)
[  353.067606][    C3]  ? update_socket_protocol+0x10/0x10
[  353.068000][    C3]  __x64_sys_connect (net/socket.c:2124)
[  353.068355][    C3]  ? lockdep_hardirqs_on (kernel/locking/lockdep.c:4472)
[  353.068996][    C3]  do_syscall_64 (arch/x86/entry/syscall_64.c:63 arch/x86/entry/syscall_64.c:94)
[  353.069355][    C3]  entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:130)
[  353.069826][    C3] RIP: 0033:0x7f518bc06d67
[  353.070469][    C3] Code: 64 89 01 48 83 c8 ff c3 66 2e 0f 1f 84 00 00 00 00 00 90 f3 0f 1e fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 2a 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 51 c3 48 83 ec 18 89 54 24 0c 48 89 34 24 89
All code
========
   0:	64 89 01             	mov    %eax,%fs:(%rcx)
   3:	48 83 c8 ff          	or     $0xffffffffffffffff,%rax
   7:	c3                   	ret
   8:	66 2e 0f 1f 84 00 00 	cs nopw 0x0(%rax,%rax,1)
   f:	00 00 00 
  12:	90                   	nop
  13:	f3 0f 1e fa          	endbr64
  17:	64 8b 04 25 18 00 00 	mov    %fs:0x18,%eax
  1e:	00 
  1f:	85 c0                	test   %eax,%eax
  21:	75 10                	jne    0x33
  23:	b8 2a 00 00 00       	mov    $0x2a,%eax
  28:	0f 05                	syscall
  2a:*	48 3d 00 f0 ff ff    	cmp    $0xfffffffffffff000,%rax		<-- trapping instruction
  30:	77 51                	ja     0x83
  32:	c3                   	ret
  33:	48 83 ec 18          	sub    $0x18,%rsp
  37:	89 54 24 0c          	mov    %edx,0xc(%rsp)
  3b:	48 89 34 24          	mov    %rsi,(%rsp)
  3f:	89                   	.byte 0x89

Code starting with the faulting instruction
===========================================
   0:	48 3d 00 f0 ff ff    	cmp    $0xfffffffffffff000,%rax
   6:	77 51                	ja     0x59
   8:	c3                   	ret
   9:	48 83 ec 18          	sub    $0x18,%rsp
   d:	89 54 24 0c          	mov    %edx,0xc(%rsp)
  11:	48 89 34 24          	mov    %rsi,(%rsp)
  15:	89                   	.byte 0x89
[  353.071817][    C3] RSP: 002b:00007fffc4714b98 EFLAGS: 00000246 ORIG_RAX: 000000000000002a
[  353.072629][    C3] RAX: ffffffffffffffda RBX: 000055e6560f2d10 RCX: 00007f518bc06d67
[  353.073192][    C3] RDX: 0000000000000010 RSI: 00007fffc4715510 RDI: 0000000000000008
[  353.073733][    C3] RBP: 00007fffc4715510 R08: 00007fffc4714ba0 R09: 0000000000000010
[  353.074554][    C3] R10: 000000000000be23 R11: 0000000000000246 R12: 0000000000000010


Finger prints:
__dev_xmit_skb:__dev_queue_xmit:__neigh_update:arp_process:__netif_receive_skb_one_core