======================================
| [  124.768495][    C1] TCP: AO hash mismatch for 10.0.254.1.7012->10.0.1.1.49486 [FP.] L3index: 0
| [  125.496015][    C3] ------------[ cut here ]------------
| [  125.496677][    C3] kernel BUG at net/core/skbuff.c:2813!
| [  125.496961][    C3] invalid opcode: 0000 [#1] PREEMPT SMP KASAN NOPTI
[  125.497580][    C3] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014
[ 125.498078][ C3] RIP: 0010:__pskb_pull_tail (net/core/skbuff.c:2813 (discriminator 1)) 
[ 125.498364][ C3] Code: e9 52 f7 ff ff 48 8b 7c 24 38 be 02 00 00 00 e8 9e 9c ff ff 48 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 cc cc cc cc 90 <0f> 0b 48 c7 44 24 38 00 00 00 00 e9 dc fa ff ff 84 c9 0f 84 4b f9
All code
========
   0:	e9 52 f7 ff ff       	jmp    0xfffffffffffff757
   5:	48 8b 7c 24 38       	mov    0x38(%rsp),%rdi
   a:	be 02 00 00 00       	mov    $0x2,%esi
   f:	e8 9e 9c ff ff       	call   0xffffffffffff9cb2
  14:	48 83 c4 58          	add    $0x58,%rsp
  18:	31 c0                	xor    %eax,%eax
  1a:	5b                   	pop    %rbx
  1b:	41 5c                	pop    %r12
  1d:	41 5d                	pop    %r13
  1f:	41 5e                	pop    %r14
  21:	41 5f                	pop    %r15
  23:	5d                   	pop    %rbp
  24:	c3                   	ret
  25:	cc                   	int3
  26:	cc                   	int3
  27:	cc                   	int3
  28:	cc                   	int3
  29:	90                   	nop
  2a:*	0f 0b                	ud2		<-- trapping instruction
  2c:	48 c7 44 24 38 00 00 	movq   $0x0,0x38(%rsp)
  33:	00 00 
  35:	e9 dc fa ff ff       	jmp    0xfffffffffffffb16
  3a:	84 c9                	test   %cl,%cl
  3c:	0f                   	.byte 0xf
  3d:	84 4b f9             	test   %cl,-0x7(%rbx)

Code starting with the faulting instruction
===========================================
   0:	0f 0b                	ud2
   2:	48 c7 44 24 38 00 00 	movq   $0x0,0x38(%rsp)
   9:	00 00 
   b:	e9 dc fa ff ff       	jmp    0xfffffffffffffaec
  10:	84 c9                	test   %cl,%cl
  12:	0f                   	.byte 0xf
  13:	84 4b f9             	test   %cl,-0x7(%rbx)
[  125.499163][    C3] RSP: 0018:ffffc90000278b68 EFLAGS: 00010282
[  125.499476][    C3] RAX: 00000000fffffff2 RBX: ffff888002b569ac RCX: 0000000000000004
[  125.499810][    C3] RDX: dffffc0000000000 RSI: 1ffff11000c38509 RDI: ffff8880061c2848
[  125.500139][    C3] RBP: ffffc90000278be8 R08: 0000000000000014 R09: ffff888002b569f4
[  125.500470][    C3] R10: ffff888002b569f8 R11: ffff888002b569f8 R12: ffff888002b56940
[  125.500801][    C3] R13: 0000000000000100 R14: 0000000000000000 R15: ffff888002b569ea
[  125.501131][    C3] FS:  00007f9d9b05d640(0000) GS:ffff888036180000(0000) knlGS:0000000000000000
[  125.501515][    C3] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  125.501792][    C3] CR2: 00007f9d9401c068 CR3: 0000000005096004 CR4: 0000000000770ef0
[  125.502125][    C3] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[  125.502456][    C3] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[  125.502790][    C3] PKRU: 55555554
[  125.502957][    C3] Call Trace:
[  125.503122][    C3]  <IRQ>
[ 125.503236][ C3] ? die (arch/x86/kernel/dumpstack.c:421 arch/x86/kernel/dumpstack.c:434 arch/x86/kernel/dumpstack.c:447) 
[ 125.503407][ C3] ? do_trap (arch/x86/kernel/traps.c:113 arch/x86/kernel/traps.c:154) 
[ 125.503574][ C3] ? __pskb_pull_tail (net/core/skbuff.c:2813 (discriminator 1)) 
[ 125.503794][ C3] ? do_error_trap (./arch/x86/include/asm/traps.h:58 arch/x86/kernel/traps.c:175) 
[ 125.504012][ C3] ? __pskb_pull_tail (net/core/skbuff.c:2813 (discriminator 1)) 
[ 125.504231][ C3] ? __pskb_pull_tail (net/core/skbuff.c:2813 (discriminator 1)) 
[ 125.504457][ C3] ? handle_invalid_op (arch/x86/kernel/traps.c:213) 
[ 125.504675][ C3] ? __pskb_pull_tail (net/core/skbuff.c:2813 (discriminator 1)) 
[ 125.504894][ C3] ? exc_invalid_op (arch/x86/kernel/traps.c:265) 
[ 125.505116][ C3] ? asm_exc_invalid_op (./arch/x86/include/asm/idtentry.h:568) 
[ 125.505341][ C3] ? __pskb_pull_tail (net/core/skbuff.c:2813 (discriminator 1)) 
[ 125.505559][ C3] ? __pskb_pull_tail (net/core/skbuff.c:2813) 
[ 125.505779][ C3] ? __netif_receive_skb_core.constprop.0 (net/core/dev.c:5498) 
[ 125.506057][ C3] tcp_v4_early_demux (./include/linux/skbuff.h:2731 ./include/linux/skbuff.h:2739 net/ipv4/tcp_ipv4.c:1966) 
[ 125.506282][ C3] ip_rcv_finish_core.constprop.0 (./include/linux/skbuff.h:2943 ./include/linux/ip.h:21 net/ipv4/ip_input.c:344) 
[ 125.506554][ C3] ? __pfx_ip_rcv (net/ipv4/ip_input.c:562) 
[ 125.506774][ C3] ip_rcv (net/ipv4/ip_input.c:448 ./include/linux/netfilter.h:422 net/ipv4/ip_input.c:569) 
[ 125.506939][ C3] ? __pfx_ip_rcv (net/ipv4/ip_input.c:562) 
[ 125.507161][ C3] __netif_receive_skb_one_core (net/core/dev.c:5610 (discriminator 4)) 
[ 125.507431][ C3] ? __pfx___netif_receive_skb_one_core (net/core/dev.c:5603) 
[ 125.507705][ C3] ? lock_acquire (./include/trace/events/lock.h:24 kernel/locking/lockdep.c:5725) 
[ 125.507924][ C3] ? process_backlog (./include/linux/rcupdate.h:298 ./include/linux/rcupdate.h:750 net/core/dev.c:6051) 
[ 125.508150][ C3] process_backlog (./include/linux/rcupdate.h:779 net/core/dev.c:6053) 
[ 125.508368][ C3] ? find_held_lock (kernel/locking/lockdep.c:5244) 
[ 125.508592][ C3] __napi_poll.constprop.0 (net/core/dev.c:6703) 
[ 125.508814][ C3] net_rx_action (net/core/dev.c:6772 net/core/dev.c:6886) 
[ 125.509053][ C3] ? __pfx_net_rx_action (net/core/dev.c:6850) 
[ 125.509272][ C3] ? hlock_class (kernel/locking/lockdep.c:223) 
[ 125.509494][ C3] ? mark_held_locks (kernel/locking/lockdep.c:4274) 
[ 125.509712][ C3] ? hrtimer_interrupt (kernel/time/hrtimer.c:1828) 
[ 125.509937][ C3] __do_softirq (kernel/softirq.c:553) 
[ 125.510160][ C3] ? __dev_queue_xmit (./include/linux/rcupdate.h:308 ./include/linux/rcupdate.h:818 net/core/dev.c:4399) 
[ 125.510377][ C3] do_softirq (kernel/softirq.c:454 kernel/softirq.c:441) 
[  125.510543][    C3]  </IRQ>
[  125.510656][    C3]  <TASK>
[ 125.510766][ C3] __local_bh_enable_ip (kernel/softirq.c:381) 
[ 125.510986][ C3] ? __dev_queue_xmit (./include/linux/rcupdate.h:308 ./include/linux/rcupdate.h:818 net/core/dev.c:4399) 
[ 125.511205][ C3] __dev_queue_xmit (net/core/dev.c:4400) 
[ 125.511429][ C3] ? lockdep_hardirqs_on_prepare.part.0 (kernel/locking/lockdep.c:4300 kernel/locking/lockdep.c:4359) 
[ 125.511704][ C3] ? __pfx___dev_queue_xmit (net/core/dev.c:4281) 
[ 125.511924][ C3] ? neigh_hh_output (./include/linux/seqlock.h:74 ./include/linux/seqlock.h:772 ./include/net/neighbour.h:496) 
[ 125.512147][ C3] ip_finish_output2 (./include/net/neighbour.h:540 net/ipv4/ip_output.c:235) 
[ 125.512368][ C3] ? hlock_class (./arch/x86/include/asm/bitops.h:227 ./arch/x86/include/asm/bitops.h:239 ./include/asm-generic/bitops/instrumented-non-atomic.h:142 kernel/locking/lockdep.c:228) 
[ 125.512590][ C3] ? __pfx_ip_finish_output2 (net/ipv4/ip_output.c:199) 
[ 125.512811][ C3] ? __ip_finish_output (./include/linux/skbuff.h:1624 ./include/linux/skbuff.h:4942 net/ipv4/ip_output.c:307) 
[ 125.513039][ C3] __ip_queue_xmit (net/ipv4/ip_output.c:535 (discriminator 4)) 
[ 125.513259][ C3] ? tcp_options_write (net/ipv4/tcp_output.c:670) 
[ 125.513484][ C3] __tcp_transmit_skb (net/ipv4/tcp_output.c:1462 (discriminator 4)) 
[ 125.513707][ C3] ? __pfx___tcp_transmit_skb (net/ipv4/tcp_output.c:1283) 
[ 125.513930][ C3] ? tcp_mtu_probe (./include/net/net_namespace.h:383 ./include/net/sock.h:648 net/ipv4/tcp_output.c:2420) 
[ 125.514152][ C3] ? mark_held_locks (kernel/locking/lockdep.c:4274) 
[ 125.514372][ C3] ? tcp_small_queue_check.isra.0 (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./include/linux/refcount.h:136 net/ipv4/tcp_output.c:2623) 
[ 125.514646][ C3] tcp_write_xmit (net/ipv4/tcp_output.c:2792) 
[ 125.514871][ C3] ? __alloc_skb (./arch/x86/include/asm/atomic.h:28 ./include/linux/atomic/atomic-arch-fallback.h:503 ./include/linux/atomic/atomic-instrumented.h:68 ./include/linux/refcount.h:125 net/core/skbuff.c:696) 
[ 125.515090][ C3] ? __pfx_tcp_write_xmit (net/ipv4/tcp_output.c:2702) 
[ 125.515314][ C3] ? skb_do_copy_data_nocache (./include/net/sock.h:2258) 
[ 125.515532][ C3] ? mark_lock (kernel/locking/lockdep.c:4656 (discriminator 3)) 
[ 125.515699][ C3] ? __pfx_skb_do_copy_data_nocache (./include/net/sock.h:2257) 
[ 125.515972][ C3] ? skb_page_frag_refill (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./include/linux/page_ref.h:67 net/core/sock.c:2908) 
[ 125.516193][ C3] __tcp_push_pending_frames (net/ipv4/tcp_output.c:2977) 
[ 125.516414][ C3] tcp_sendmsg_locked (net/ipv4/tcp.c:1309) 
[ 125.516638][ C3] ? __lock_release (kernel/locking/lockdep.c:5430) 
[ 125.516858][ C3] ? tcp_sendmsg (net/ipv4/tcp.c:1341) 
[ 125.517082][ C3] ? __pfx_tcp_sendmsg_locked (net/ipv4/tcp.c:1039) 
[ 125.517297][ C3] ? hlock_class (./arch/x86/include/asm/bitops.h:227 ./arch/x86/include/asm/bitops.h:239 ./include/asm-generic/bitops/instrumented-non-atomic.h:142 kernel/locking/lockdep.c:228) 
[ 125.517515][ C3] ? mark_lock (kernel/locking/lockdep.c:4656 (discriminator 3)) 
[ 125.517682][ C3] tcp_sendmsg (net/ipv4/tcp.c:1342) 
[ 125.517846][ C3] __sys_sendto (net/socket.c:730 net/socket.c:745 net/socket.c:2191) 
[ 125.518063][ C3] ? __fget_files (./include/linux/rcupdate.h:308 ./include/linux/rcupdate.h:783 fs/file.c:1037) 
[ 125.518284][ C3] ? __pfx___sys_sendto (net/socket.c:2161) 
[ 125.518501][ C3] ? __pfx_do_sock_getsockopt (net/socket.c:2351) 
[ 125.518720][ C3] ? __fget_files (fs/file.c:1040) 
[ 125.518940][ C3] ? fput (./arch/x86/include/asm/atomic64_64.h:61 ./include/linux/atomic/atomic-arch-fallback.h:4382 ./include/linux/atomic/atomic-long.h:1563 ./include/linux/atomic/atomic-instrumented.h:4504 fs/file_table.c:427) 
[ 125.519120][ C3] ? __sys_getsockopt (net/socket.c:2406) 
[ 125.519335][ C3] ? __pfx___sys_getsockopt (net/socket.c:2392) 
[ 125.519560][ C3] __x64_sys_sendto (net/socket.c:2199) 
[ 125.519776][ C3] ? lockdep_hardirqs_on_prepare.part.0 (kernel/locking/lockdep.c:4300 kernel/locking/lockdep.c:4359) 
[ 125.520045][ C3] do_syscall_64 (arch/x86/entry/common.c:52 arch/x86/entry/common.c:83) 
[ 125.520265][ C3] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:129) 
[  125.520541][    C3] RIP: 0033:0x7f9d9b1b074e
[ 125.520764][ C3] Code: 4c 24 1c e8 b4 c4 f4 ff 44 8b 54 24 1c 8b 3c 24 45 31 c9 41 89 c4 48 8b 54 24 10 48 8b 74 24 08 45 31 c0 b8 2c 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 32 44 89 e7 48 89 04 24 e8 fe c4 f4 ff 48 8b
All code
========
   0:	4c 24 1c             	rex.WR and $0x1c,%al
   3:	e8 b4 c4 f4 ff       	call   0xfffffffffff4c4bc
   8:	44 8b 54 24 1c       	mov    0x1c(%rsp),%r10d
   d:	8b 3c 24             	mov    (%rsp),%edi
  10:	45 31 c9             	xor    %r9d,%r9d
  13:	41 89 c4             	mov    %eax,%r12d
  16:	48 8b 54 24 10       	mov    0x10(%rsp),%rdx
  1b:	48 8b 74 24 08       	mov    0x8(%rsp),%rsi
  20:	45 31 c0             	xor    %r8d,%r8d
  23:	b8 2c 00 00 00       	mov    $0x2c,%eax
  28:	0f 05                	syscall
  2a:*	48 3d 00 f0 ff ff    	cmp    $0xfffffffffffff000,%rax		<-- trapping instruction
  30:	77 32                	ja     0x64
  32:	44 89 e7             	mov    %r12d,%edi
  35:	48 89 04 24          	mov    %rax,(%rsp)
  39:	e8 fe c4 f4 ff       	call   0xfffffffffff4c53c
  3e:	48                   	rex.W
  3f:	8b                   	.byte 0x8b

Code starting with the faulting instruction
===========================================
   0:	48 3d 00 f0 ff ff    	cmp    $0xfffffffffffff000,%rax
   6:	77 32                	ja     0x3a
   8:	44 89 e7             	mov    %r12d,%edi
   b:	48 89 04 24          	mov    %rax,(%rsp)
   f:	e8 fe c4 f4 ff       	call   0xfffffffffff4c512
  14:	48                   	rex.W
  15:	8b                   	.byte 0x8b
[  125.521534][    C3] RSP: 002b:00007f9d9b05c480 EFLAGS: 00000246 ORIG_RAX: 000000000000002c
[  125.521866][    C3] RAX: ffffffffffffffda RBX: 0000000000000008 RCX: 00007f9d9b1b074e
[  125.522198][    C3] RDX: 0000000000000064 RSI: 00007f9d9b05c5a0 RDI: 0000000000000008
[  125.522522][    C3] RBP: 00007f9d9b05c590 R08: 0000000000000000 R09: 0000000000000000
[  125.522850][    C3] R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000000


Finger prints:
__pskb_pull_tail:tcp_v4_early_demux:ip_rcv:__netif_receive_skb_one_core