make -C tools/testing/selftests TARGETS="net/mptcp" TEST_PROGS=mptcp_join..sh TEST_GEN_PROGS="" run_tests make: Entering directory '/home/virtme/testing-11/tools/testing/selftests' make[1]: Entering directory '/home/virtme/testing-11/tools/testing/selftests/net/mptcp' make[1]: Nothing to be done for 'all'. make[1]: Leaving directory '/home/virtme/testing-11/tools/testing/selftests/net/mptcp' make[1]: Entering directory '/home/virtme/testing-11/tools/testing/selftests/net/mptcp' TAP version 13 1..1 # timeout set to 1800 # selftests: net/mptcp: mptcp_join.sh # 001 no JOIN # join Rx [ OK ] # join Tx [ OK ] # 002 single subflow, limited by client # join Rx [ OK ] # join Tx [ OK ] # 003 single subflow, limited by server # join Rx [ OK ] # join Tx [ OK ] # 004 single subflow # join Rx [ OK ] # join Tx [ OK ] # 005 multiple subflows # join Rx [ OK ] # join Tx [ OK ] # 006 multiple subflows, limited by server # join Rx [ OK ] # join Tx [ OK ] # 007 single subflow, dev # join Rx [ OK ] # join Tx [ OK ] # 008 no MPC reuse with single endpoint # join Rx [ OK ] # join Tx [ OK ] # 009 multi subflows, with failing subflow # join Rx [ OK ] # join Tx [ OK ] # 010 multi subflows, with subflow timeout # join Rx [ OK ] # join Tx [ OK ] # 011 multi subflows, fair usage on close # join Rx [ OK ] # join Tx [ OK ] # 012 unused signal address # join Rx [ OK ] # join Tx [ OK ] # add addr tx [ OK ] # add addr echo tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 013 signal address # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 014 subflow and signal # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 015 subflow and signal together # join Rx [ OK ] # join Tx [ OK ] # add addr rx server [ OK ] # add addr echo rx client [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # reset tx client [ OK ] # reset rx server [ OK ] # reset tx server [ OK ] # reset rx client [ OK ] # 016 multiple subflows and signal # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 017 signal addresses # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 018 signal invalid addresses # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 019 signal addresses race test # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 020 multiple flows, signal, link failure # Info: Test file (size 13256 KB) for client # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # stale [ OK ] # 021 multi flows, signal, bidi, link fail # Info: Test file (size 26624 KB) for server # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # stale [ OK ] # 022 backup subflow unused, link failure # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # link usage [ OK ] # 023 backup flow used, multi links fail # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # stale [ OK ] # link usage [ OK ] # 024 backup flow used, bidi, link failure # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # stale [ OK ] # link usage [ OK ] # 025 signal address, ADD_ADDR timeout # join Rx [ OK ] # join Tx [ OK ] # add addr tx [ OK ] # add addr echo tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 026 signal address, ADD_ADDR6 timeout # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 027 signal addresses, ADD_ADDR timeout # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 028 invalid address, ADD_ADDR timeout # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 029 remove single subflow # join Rx [ OK ] # join Tx [ OK ] # rm addr tx client [ OK ] # rm addr rx server [ OK ] # rm subflow client [ OK ] # reset tx server [ OK ] # reset rx client [ OK ] # 030 remove multiple subflows # join Rx [ OK ] # join Tx [ OK ] # rm addr rx server [ OK ] # rm subflow client [ OK ] # reset tx server [ OK ] # reset rx client [ OK ] # 031 remove single address # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # rm addr rx client [ OK ] # rm subflow server [ OK ] # reset tx server [ OK ] # reset rx client [ OK ] # 032 remove subflow and signal # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # rm addr rx server [ OK ] # rm subflow client [ OK ] # reset tx server [ OK ] # reset rx client [ OK ] # 033 remove subflows and signal # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # rm addr rx server [ OK ] # rm subflow client [ OK ] # reset tx server [ OK ] # reset rx client [ OK ] # 034 remove addresses # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # rm addr rx client [ OK ] # rm subflow server [ OK ] # reset tx server [ OK ] # reset rx client [ OK ] # 035 remove invalid addresses # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # rm addr rx client [ OK ] # rm subflow server [ OK ] # reset tx server [ OK ] # reset rx client [ OK ] # 036 flush subflows and signal # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # rm addr rx client [ OK ] # rm subflow server [ OK ] # reset tx server [ OK ] # reset rx client [ OK ] # 037 flush subflows # join Rx [ OK ] # join Tx [ OK ] # rm addr tx client [ OK ] # rm addr rx server [ OK ] # rm subflow client [ OK ] # reset tx server [ OK ] # reset rx client [ OK ] # 038 flush addresses # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # rm addr rx client [ OK ] # rm subflow server [ OK ] # reset tx server [ OK ] # reset rx client [ OK ] # 039 flush invalid addresses # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # rm addr rx client [ OK ] # rm subflow server [ OK ] # reset tx server [ OK ] # reset rx client [ OK ] # 040 remove id 0 subflow # join Rx [ OK ] # join Tx [ OK ] # rm addr rx server [ OK ] # rm subflow client [ OK ] # reset tx server [ OK ] # reset rx client [ OK ] # 041 remove id 0 address # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # rm addr rx client [ OK ] # rm subflow server [ OK ] # reset tx client [ OK ] # reset rx server [ OK ] # 042 add single subflow # currently established: 1 [ OK ] # join Rx [ OK ] # join Tx [ OK ] # currently established: 0 [ OK ] # 043 add signal address # currently established: 1 [ OK ] # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # currently established: 0 [ OK ] # 044 add multiple subflows # currently established: 1 [ OK ] # join Rx [ OK ] # join Tx [ OK ] # currently established: 0 [ OK ] # 045 add multiple subflows IPv6 # currently established: 1 [ OK ] # join Rx [ OK ] # join Tx [ OK ] # currently established: 0 [ OK ] # 046 add multiple addresses IPv6 # currently established: 1 [ OK ] # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # currently established: 0 [ OK ] # 047 single subflow IPv6 # join Rx [ OK ] # join Tx [ OK ] # 048 unused signal address IPv6 # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 049 single address IPv6 # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 050 remove single address IPv6 # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # rm addr rx client [ OK ] # rm subflow server [ OK ] # 051 remove subflow and signal IPv6 # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # rm addr rx server [ OK ] # rm subflow client [ OK ] # 052 single subflow IPv4-mapped # join Rx [ OK ] # join Tx [ OK ] # 053 signal address IPv4-mapped # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 054 single subflow v4-map-v6 # join Rx [ OK ] # join Tx [ OK ] # 055 signal address v4-map-v6 # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 056 single subflow v6-map-v4 # join Rx [ OK ] # join Tx [ OK ] # 057 signal address v6-map-v4 # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 058 no JOIN with diff families v4-v6 # join Rx [ OK ] # join Tx [ OK ] # 059 no JOIN with diff families v4-v6-2 # join Rx [ OK ] # join Tx [ OK ] # 060 no JOIN with diff families v6-v4 # join Rx [ OK ] # join Tx [ OK ] # 061 IPv4 sockets do not use IPv6 addresses # join Rx [ OK ] # join Tx [ OK ] # 062 simult IPv4 and IPv6 subflows # join Rx [ OK ] # join Tx [ OK ] # 063 simult IPv4 and IPv6 subflows, fullmesh 1x1 # join Rx [ OK ] # join Tx [ OK ] # 064 simult IPv4 and IPv6 subflows, fullmesh 2x2 # join Rx [ OK ] # join Tx [ OK ] # 065 single subflow, backup # join Rx [ OK ] # join Tx [ OK ] # mp_prio tx server [ OK ] # mp_prio rx client [ OK ] # syn backup [ OK ] # synack backup [ OK ] # 066 single address, backup # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # mp_prio tx server [ OK ] # mp_prio rx client [ OK ] # syn backup [ OK ] # synack backup [ OK ] # 067 single address, switch to backup # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # mp_prio tx server [ OK ] # mp_prio rx client [ OK ] # syn backup [ OK ] # synack backup [ OK ] # 068 single address with port, backup # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # mp_prio tx server [ OK ] # mp_prio rx client [ OK ] # syn backup [ OK ] # synack backup [ OK ] # 069 mpc backup # join Rx [ OK ] # join Tx [ OK ] # mp_prio tx server [ OK ] # mp_prio rx client [ OK ] # syn backup [ OK ] # synack backup [ OK ] # 070 mpc backup both sides # join Rx [ OK ] # join Tx [ OK ] # mp_prio tx server [ OK ] # mp_prio rx client [ OK ] # syn backup [ OK ] # synack backup [ OK ] # 071 mpc switch to backup # join Rx [ OK ] # join Tx [ OK ] # mp_prio tx server [ OK ] # mp_prio rx client [ OK ] # syn backup [ OK ] # synack backup [ OK ] # 072 mpc switch to backup both sides # join Rx [ OK ] # join Tx [ OK ] # mp_prio tx server [ OK ] # mp_prio rx client [ OK ] # syn backup [ OK ] # synack backup [ OK ] # 073 signal address with port # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # add addr rx with port [ OK ] # syn rx port [ OK ] # synack rx port [ OK ] # ack rx port [ OK ] # syn rx port mismatch [ OK ] # ack rx port mismatch [ OK ] # 074 subflow and signal with port # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # add addr rx with port [ OK ] # syn rx port [ OK ] # synack rx port [ OK ] # ack rx port [ OK ] # syn rx port mismatch [ OK ] # ack rx port mismatch [ OK ] # 075 remove single address with port # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # add addr rx with port [ OK ] # syn rx port [ OK ] # synack rx port [ OK ] # ack rx port [ OK ] # syn rx port mismatch [ OK ] # ack rx port mismatch [ OK ] # rm addr rx client [ OK ] # rm subflow server [ OK ] # LISTENER_CREATED 10.0.2.1:10100 [ OK ] # LISTENER_CLOSED 10.0.2.1:10100 [ OK ] # 076 remove subflow and signal with port # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # add addr rx with port [ OK ] # syn rx port [ OK ] # synack rx port [ OK ] # ack rx port [ OK ] # syn rx port mismatch [ OK ] # ack rx port mismatch [ OK ] # rm addr rx server [ OK ] # rm subflow client [ OK ] # 077 flush subflows and signal with port # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # rm addr rx client [ OK ] # rm subflow server [ OK ] # 078 multiple addresses with port # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # add addr rx with port [ OK ] # syn rx port [ OK ] # synack rx port [ OK ] # ack rx port [ OK ] # syn rx port mismatch [ OK ] # ack rx port mismatch [ OK ] # 079 multiple addresses with ports # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # add addr rx with port [ OK ] # syn rx port [ OK ] # synack rx port [ OK ] # ack rx port [ OK ] # syn rx port mismatch [ OK ] # ack rx port mismatch [ OK ] # 080 port-based signal endpoint must not accept mpc # Connect [ OK ] # Attempts [ OK ] # 081 single subflow with syn cookies # join Rx [ OK ] # join Tx [ OK ] # 082 multiple subflows with syn cookies # join Rx [ OK ] # join Tx [ OK ] # 083 subflows limited by server w cookies # join Rx [ OK ] # join Tx [ OK ] # 084 signal address with syn cookies # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 085 subflow and signal w cookies # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 086 subflows and signal w. cookies # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 087 checksum test 0 0 # join Rx [ OK ] # join Tx [ OK ] # checksum server [ OK ] # checksum client [ OK ] # fail tx server [ OK ] # fail rx client [ OK ] # reset tx server [ OK ] # reset rx client [ OK ] # infi tx client [ OK ] # infi rx server [ OK ] # 088 checksum test 1 1 # join Rx [ OK ] # join Tx [ OK ] # checksum server [ OK ] # checksum client [ OK ] # fail tx server [ OK ] # fail rx client [ OK ] # reset tx server [ OK ] # reset rx client [ OK ] # infi tx client [ OK ] # infi rx server [ OK ] # 089 checksum test 0 1 # join Rx [ OK ] # join Tx [ OK ] # checksum server [ OK ] # checksum client [ OK ] # fail tx server [ OK ] # fail rx client [ OK ] # reset tx server [ OK ] # reset rx client [ OK ] # infi tx client [ OK ] # infi rx server [ OK ] # 090 checksum test 1 0 # join Rx [ OK ] # join Tx [ OK ] # checksum server [ OK ] # checksum client [ OK ] # fail tx server [ OK ] # fail rx client [ OK ] # reset tx server [ OK ] # reset rx client [ OK ] # infi tx client [ OK ] # infi rx server [ OK ] # 091 single subflow allow join id0 ns1 # join Rx [ OK ] # join Tx [ OK ] # 092 single subflow allow join id0 ns2 # join Rx [ OK ] # join Tx [ OK ] # 093 signal address allow join id0 ns1 # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 094 signal address allow join id0 ns2 # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 095 subflow and address allow join id0 1 # join Rx [ OK ] # join Tx [ OK ] # 096 subflow and address allow join id0 2 # join Rx [ OK ] # join Tx [ OK ] # 097 fullmesh test 2x1 # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 098 fullmesh test 1x1 # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 099 fullmesh test 1x2 # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 100 fullmesh test 1x2, limited # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 101 set fullmesh flag test # join Rx [ OK ] # join Tx [ OK ] # rm addr rx server [ OK ] # rm subflow client [ OK ] # 102 set nofullmesh flag test # join Rx [ OK ] # join Tx [ OK ] # rm addr rx server [ OK ] # rm subflow client [ OK ] # 103 set backup,fullmesh flags test # join Rx [ OK ] # join Tx [ OK ] # mp_prio tx server [ OK ] # mp_prio rx client [ OK ] # syn backup [ OK ] # synack backup [ OK ] # rm addr rx server [ OK ] # rm subflow client [ OK ] # 104 set nobackup,nofullmesh flags test # join Rx [ OK ] # join Tx [ OK ] # mp_prio tx server [ OK ] # mp_prio rx client [ OK ] # syn backup [ OK ] # synack backup [ OK ] # rm addr rx server [ OK ] # rm subflow client [ OK ] # 105 fastclose test # Info: Test file (size 1024 KB) for client # Info: Test file (size 1024 KB) for server # copyfd_io_poll: poll timed out (events: POLLIN 0, POLLOUT 4) # [IGNO] (flaky) client exit code 124, server 0 # Server ns stats # TcpPassiveOpens 1 0.0 # TcpEstabResets 1 0.0 # TcpInSegs 18 0.0 # TcpOutSegs 94 0.0 # TcpExtTCPPureAcks 14 0.0 # TcpExtTCPLossProbes 3 0.0 # TcpExtTCPOrigDataSent 90 0.0 # TcpExtTCPDelivered 90 0.0 # MPTcpExtMPCapableSYNRX 1 0.0 # MPTcpExtMPCapableACKRX 1 0.0 # MPTcpExtMPFastcloseRx 1 0.0 # MPTcpExtMPRstRx 1 0.0 # Client ns stats # TcpActiveOpens 1 0.0 # TcpEstabResets 1 0.0 # TcpInSegs 23 0.0 # TcpOutSegs 18 0.0 # TcpOutRsts 1 0.0 # TcpExtDelayedACKs 3 0.0 # TcpExtTCPPureAcks 3 0.0 # TcpExtTCPToZeroWindowAdv 1 0.0 # TcpExtTCPOrigDataSent 2 0.0 # TcpExtTCPDelivered 3 0.0 # MPTcpExtMPCapableSYNTX 1 0.0 # MPTcpExtMPCapableSYNACKRX 1 0.0 # MPTcpExtMPFastcloseTx 1 0.0 # MPTcpExtMPRstTx 1 0.0 # # netns ns1-T6H5ah (listener) socket stat for 10104: # Netid State Recv-Q Send-Q Local Address:Port Peer Address:Port # TcpPassiveOpens 1 0.0 # TcpEstabResets 1 0.0 # TcpInSegs 18 0.0 # TcpOutSegs 94 0.0 # TcpExtTCPPureAcks 14 0.0 # TcpExtTCPLossProbes 3 0.0 # TcpExtTCPOrigDataSent 90 0.0 # TcpExtTCPDelivered 90 0.0 # MPTcpExtMPCapableSYNRX 1 0.0 # MPTcpExtMPCapableACKRX 1 0.0 # MPTcpExtMPFastcloseRx 1 0.0 # MPTcpExtMPRstRx 1 0.0 # # netns ns2-xNM3NC (connector) socket stat for 10104: # Netid State Recv-Q Send-Q Local Address:Port Peer Address:Port # TcpActiveOpens 1 0.0 # TcpEstabResets 1 0.0 # TcpInSegs 23 0.0 # TcpOutSegs 18 0.0 # TcpOutRsts 1 0.0 # TcpExtDelayedACKs 3 0.0 # TcpExtTCPPureAcks 3 0.0 # TcpExtTCPToZeroWindowAdv 1 0.0 # TcpExtTCPOrigDataSent 2 0.0 # TcpExtTCPDelivered 3 0.0 # MPTcpExtMPCapableSYNTX 1 0.0 # MPTcpExtMPCapableSYNACKRX 1 0.0 # MPTcpExtMPFastcloseTx 1 0.0 # MPTcpExtMPRstTx 1 0.0 # join Rx [ OK ] # join Tx [ OK ] # fast close tx client [ OK ] # fast close rx server [ OK ] # reset tx client [ OK ] # reset rx server [ OK ] # 106 fastclose server test # Info: Test file (size 1024 KB) for client # Info: Test file (size 1024 KB) for server # join Rx [ OK ] # join Tx [ OK ] # fast close tx server [ OK ] # fast close rx client [ OK ] # reset tx server [ OK ] # reset rx client [ OK ] # 107 Infinite map # Info: Test file (size 128 KB) for client # Info: Test file (size 128 KB) for server # copyfd_io_poll: poll timed out (events: POLLIN 1, POLLOUT 0) # [IGNO] (flaky) client exit code 124, server 0 # Server ns stats # TcpPassiveOpens 1 0.0 # TcpInSegs 21 0.0 # TcpOutSegs 113 0.0 # TcpRetransSegs 16 0.0 # TcpExtTCPPureAcks 13 0.0 # TcpExtTCPLossUndo 9 0.0 # TcpExtTCPTimeouts 9 0.0 # TcpExtTCPLossProbes 10 0.0 # TcpExtTCPBacklogCoalesce 1 0.0 # TcpExtTCPDSACKRecv 9 0.0 # TcpExtTCPSackShiftFallback 9 0.0 # TcpExtTCPOrigDataSent 108 0.0 # TcpExtTCPDelivered 111 0.0 # TcpExtTcpTimeoutRehash 9 0.0 # TcpExtTCPDSACKRecvSegs 9 0.0 # MPTcpExtMPCapableSYNRX 1 0.0 # MPTcpExtMPCapableACKRX 1 0.0 # MPTcpExtMPTCPRetrans 7 0.0 # MPTcpExtDataCsumErr 1 0.0 # MPTcpExtMPFailTx 1 0.0 # MPTcpExtMPFailRx 1 0.0 # Client ns stats # TcpActiveOpens 1 0.0 # TcpInSegs 41 0.0 # TcpOutSegs 333 0.0 # TcpExtOutOfWindowIcmps 210 0.0 # TcpExtDelayedACKLost 16 0.0 # TcpExtTCPPureAcks 4 0.0 # TcpExtTCPDSACKOldSent 16 0.0 # TcpExtTCPOrigDataSent 19 0.0 # TcpExtTCPDelivered 20 0.0 # MPTcpExtMPCapableSYNTX 1 0.0 # MPTcpExtMPCapableSYNACKRX 1 0.0 # MPTcpExtInfiniteMapTx 1 0.0 # MPTcpExtMPFailTx 1 0.0 # MPTcpExtMPFailRx 1 0.0 # # netns ns1-T6H5ah (listener) socket stat for 10106: # Failed to find cgroup2 mount # Failed to find cgroup2 mount # Failed to find cgroup2 mount # Netid State Recv-Q Send-Q Local Address:Port Peer Address:Port # tcp ESTAB 0 8192 [::ffff:10.0.1.1]:10106 [::ffff:10.0.1.2]:34206 timer:(on,12sec,0) ino:0 sk:2 cgroup:unreachable:1 <-> # skmem:(r0,rb131072,t0,tb267264,f2432,w9856,o0,bl0,d0) ts sack cubic wscale:7,7 rto:24339 rtt:3442.69/5223.86 ato:40 mss:1448 pmtu:1500 rcvmss:1420 advmss:1448 cwnd:58 bytes_sent:166912 bytes_retrans:20096 bytes_acked:138624 bytes_received:24576 segs_out:128 segs_in:33 data_segs_out:124 data_segs_in:19 send 195159bps lastsnd:12290 lastrcv:60912 lastack:19458 pacing_rate 390312bps delivery_rate 4528bps delivered:112 busy:60040ms unacked:6 retrans:0/16 dsack_dups:9 rcv_space:14600 rcv_ssthresh:90062 minrtt:0.054 snd_wnd:263936 rcv_wnd:90112 rehash:9 tcp-ulp-mptcp flags:Mec token:0000(id:0)/e5f7e30a(id:0) seq:88fc61967461aea sfseq:4001 ssnoff:adea7e69 maplen:2000 # mptcp FIN-WAIT-1 0 0 [::ffff:10.0.1.1]:10106 [::ffff:10.0.1.2]:34206 timer:(keepalive,29sec,0) ino:0 sk:3 cgroup:unreachable:1 --- # skmem:(r0,rb131072,t0,tb271360,f3548,w123428,o0,bl0,d0) subflows_max:2 remote_key token:e5f7e30a write_seq:5260406683719685271 snd_una:5260406683719562362 rcv_nxt:616929486382185194 csum_enabled:1 bytes_retrans:57344 bytes_sent:89472 bytes_received:24576 bytes_acked:8192 subflows_total:1 last_data_sent:60707 last_data_recv:60913 last_ack_recv:60913 # TcpPassiveOpens 1 0.0 # TcpInSegs 21 0.0 # TcpOutSegs 113 0.0 # TcpRetransSegs 16 0.0 # TcpExtTCPPureAcks 13 0.0 # TcpExtTCPLossUndo 9 0.0 # TcpExtTCPTimeouts 9 0.0 # TcpExtTCPLossProbes 10 0.0 # TcpExtTCPBacklogCoalesce 1 0.0 # TcpExtTCPDSACKRecv 9 0.0 # TcpExtTCPSackShiftFallback 9 0.0 # TcpExtTCPOrigDataSent 108 0.0 # TcpExtTCPDelivered 111 0.0 # TcpExtTcpTimeoutRehash 9 0.0 # TcpExtTCPDSACKRecvSegs 9 0.0 # MPTcpExtMPCapableSYNRX 1 0.0 # MPTcpExtMPCapableACKRX 1 0.0 # MPTcpExtMPTCPRetrans 7 0.0 # MPTcpExtDataCsumErr 1 0.0 # MPTcpExtMPFailTx 1 0.0 # MPTcpExtMPFailRx 1 0.0 # # netns ns2-xNM3NC (connector) socket stat for 10106: # Failed to find cgroup2 mount # Failed to find cgroup2 mount # Failed to find cgroup2 mount # Netid State Recv-Q Send-Q Local Address:Port Peer Address:Port # tcp FIN-WAIT-1 0 106525 10.0.1.2:34206 10.0.1.1:10106 timer:(persist,484ms,5) ino:0 sk:1001 cgroup:unreachable:1 <-- # skmem:(r0,rb1343060,t0,tb129024,f4004,w110684,o0,bl0,d16) ts sack cubic wscale:7,7 rto:201 backoff:5 rtt:0.076/0.027 ato:40 mss:1448 pmtu:1500 rcvmss:1420 advmss:1448 cwnd:28 bytes_sent:429084 bytes_acked:24577 bytes_received:146816 segs_out:333 segs_in:129 data_segs_out:319 data_segs_in:124 send 4267789474bps lastsnd:5948 lastrcv:19404 lastack:12348 pacing_rate 8521563216bps delivery_rate 1148826440bps delivered:20 busy:60970ms rcv_rtt:459.747 rcv_space:14480 rcv_ssthresh:280238 notsent:106525 minrtt:0.037 snd_wnd:90112 rcv_wnd:280320 tcp-ulp-mptcp flags:Mmec token:0000(id:0)/ebc2d4c1(id:0) seq:4900b8020d33f1fa sfseq:21d81 ssnoff:7ecf67c7 maplen:2000 # mptcp FIN-WAIT-2 0 0 10.0.1.2:34206 10.0.1.1:10106 timer:(keepalive,59sec,0) ino:0 sk:1002 cgroup:unreachable:1 --- # skmem:(r0,rb1343060,t0,tb133120,f0,w0,o0,bl0,d0) subflows_max:2 fallback remote_key token:ebc2d4c1 write_seq:616929486382291719 snd_una:616929486382291719 rcv_nxt:5260406683719700986 csum_enabled:1 bytes_sent:131100 bytes_received:146816 bytes_acked:73728 last_data_sent:60967 last_data_recv:19405 last_ack_recv:60971 # TcpActiveOpens 1 0.0 # TcpInSegs 41 0.0 # TcpOutSegs 333 0.0 # TcpExtOutOfWindowIcmps 210 0.0 # TcpExtDelayedACKLost 16 0.0 # TcpExtTCPPureAcks 4 0.0 # TcpExtTCPDSACKOldSent 16 0.0 # TcpExtTCPOrigDataSent 19 0.0 # TcpExtTCPDelivered 20 0.0 # MPTcpExtMPCapableSYNTX 1 0.0 # MPTcpExtMPCapableSYNACKRX 1 0.0 # MPTcpExtInfiniteMapTx 1 0.0 # MPTcpExtMPFailTx 1 0.0 # MPTcpExtMPFailRx 1 0.0 # Info: 5 corrupted pkts # join Rx [ OK ] # join Tx [ OK ] # checksum server [ OK ] # checksum client [ OK ] # fail tx server [ OK ] # fail rx client [ OK ] # reset tx server [ OK ] # reset rx client [ OK ] # infi tx client [ OK ] # infi rx server [IGNO] (flaky) got 0 infinite map[s] RX expected 1 # fail tx client [ OK ] # fail rx server [ OK ] # 108 MP_FAIL MP_RST # Info: Test file (size 1024 KB) for client # Info: Test file (size 1024 KB) for server # Info: 1 corrupted pkts # join Rx [ OK ] # join Tx [ OK ] # checksum server [ OK ] # checksum client [ OK ] # fail tx server [ OK ] # fail rx client [ OK ] # reset tx server [ OK ] # reset rx client [ OK ] # infi tx client [ OK ] # infi rx server [ OK ] # 109 userspace pm type prevents add_addr # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 110 userspace pm no echo w/o daemon # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # 111 userspace pm type rejects join # join Rx [ OK ] # join Tx [ OK ] # 112 userspace pm type does not send join # join Rx [ OK ] # join Tx [ OK ] # 113 userspace pm type prevents mp_prio # join Rx [ OK ] # join Tx [ OK ] # mp_prio tx server [ OK ] # mp_prio rx client [ OK ] # syn backup [ OK ] # synack backup [ OK ] # 114 userspace pm type prevents rm_addr # join Rx [ OK ] # join Tx [ OK ] # rm addr rx server [ OK ] # rm subflow client [ OK ] # 115 userspace pm add & remove address # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # mptcp_info subflows=2:2 [ OK ] # mptcp_info subflows_total=3:3 [ OK ] # mptcp_info add_addr_signal=2:2 [ OK ] # dump addrs signal [ OK ] # get id 10 addr [ OK ] # get id 20 addr [ OK ] # dump addrs after rm_sf 10 [ OK ] # dump addrs after rm_addr 20 [ OK ] # rm addr rx client [ OK ] # rm subflow server [ OK ] # mptcp_info subflows=0:0 [ OK ] # mptcp_info subflows_total=1:1 [ OK ] # 116 userspace pm create destroy subflow # join Rx [ OK ] # join Tx [ OK ] # mptcp_info subflows=1:1 [ OK ] # mptcp_info subflows_total=2:2 [ OK ] # dump addrs subflow [ OK ] # get id 20 addr [ OK ] # dump addrs after rm_sf 20 [ OK ] # rm addr rx server [ OK ] # rm subflow client [ OK ] # mptcp_info subflows=0:0 [ OK ] # mptcp_info subflows_total=1:1 [ OK ] # 117 userspace pm create id 0 subflow # mptcp_info subflows=0:0 [ OK ] # mptcp_info subflows_total=1:1 [ OK ] # dump addrs id 0 subflow [ OK ] # join Rx [ OK ] # join Tx [ OK ] # mptcp_info subflows=1:1 [ OK ] # mptcp_info subflows_total=2:2 [ OK ] # 118 userspace pm remove initial subflow # join Rx [ OK ] # join Tx [ OK ] # mptcp_info subflows=1:1 [ OK ] # mptcp_info subflows_total=2:2 [ OK ] # rm addr rx server [ OK ] # rm subflow client [ OK ] # reset tx client [ OK ] # reset rx server [ OK ] # mptcp_info subflows=1:1 [ OK ] # mptcp_info subflows_total=1:1 [ OK ] # 119 userspace pm send RM_ADDR for ID 0 # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # mptcp_info subflows=1:1 [ OK ] # mptcp_info subflows_total=2:2 [ OK ] # mptcp_info add_addr_signal=1:1 [ OK ] # rm addr rx client [ OK ] # rm subflow server [ OK ] # reset tx client [ OK ] # reset rx server [ OK ] # mptcp_info subflows=1:1 [ OK ] # mptcp_info subflows_total=1:1 [ OK ] # 120 implicit EP # creation [ OK ] # mptcp_info subflows=1:1 [ OK ] # mptcp_info add_addr_signal=1:1 [ OK ] # ID change is prevented [ OK ] # modif is allowed [ OK ] # 121 delete and re-add # Info: Test file (size 4 KB) for client # Info: Test file (size 4 KB) for server # creation [ OK ] # before delete id 2 [ OK ] # mptcp_info subflows=1:1 [ OK ] # after delete id 2 [ OK ] # mptcp_info subflows=0:0 [ OK ] # after re-add id 2 [ OK ] # mptcp_info subflows=1:1 [ OK ] # after new reject [ OK ] # mptcp_info subflows=1:1 [ OK ] # after no reject [ OK ] # mptcp_info subflows=2:2 [ OK ] # after delete id 0 (1) [ OK ] # mptcp_info subflows=2:2 [ OK ] # after re-add id 0 (1) [ OK ] # mptcp_info subflows=3:3 [ OK ] # after delete id 0 (2) [ OK ] # mptcp_info subflows=2:2 [ OK ] # after re-add id 0 (2) [ OK ] # mptcp_info subflows=3:3 [ OK ] # after delete id 0 (3) [ OK ] # mptcp_info subflows=2:2 [ OK ] # after re-add id 0 (3) [ OK ] # mptcp_info subflows=3:3 [ OK ] # event ns1 LISTENER_CREATED (1) [ OK ] # event ns1 CREATED (1) [ OK ] # event ns1 ESTABLISHED (1) [ OK ] # event ns1 ANNOUNCED (0) [ OK ] # event ns1 REMOVED (4) [ OK ] # event ns1 SUB_ESTABLISHED (6) [ OK ] # event ns1 SUB_CLOSED (4) [ OK ] # event ns2 CREATED (1) [ OK ] # event ns2 ESTABLISHED (1) [ OK ] # event ns2 ANNOUNCED (0) [ OK ] # event ns2 REMOVED (0) [ OK ] # event ns2 SUB_ESTABLISHED (6) [ OK ] # event ns2 SUB_CLOSED (5) [ OK ] # join Rx [ OK ] # join Tx [ OK ] # rm addr rx server [ OK ] # rm subflow client [ OK ] # 122 delete re-add signal # Info: Test file (size 4 KB) for client # Info: Test file (size 4 KB) for server # creation [ OK ] # before delete [ OK ] # mptcp_info subflows=1:1 [ OK ] # after delete [ OK ] # mptcp_info subflows=0:0 [ OK ] # after re-add [ OK ] # mptcp_info subflows=2:2 [ OK ] # after delete ID 0 [ OK ] # mptcp_info subflows=2:2 [ OK ] # after re-add ID 0 [ OK ] # mptcp_info subflows=3:3 [ OK ] # after re-delete ID 0 [ OK ] # mptcp_info subflows=2:2 [ OK ] # after re-re-add ID 0 [ OK ] # mptcp_info subflows=3:3 [ OK ] # event ns1 LISTENER_CREATED (1) [ OK ] # event ns1 CREATED (1) [ OK ] # event ns1 ESTABLISHED (1) [ OK ] # event ns1 ANNOUNCED (0) [ OK ] # event ns1 REMOVED (0) [ OK ] # event ns1 SUB_ESTABLISHED (5) [ OK ] # event ns1 SUB_CLOSED (3) [ OK ] # event ns2 CREATED (1) [ OK ] # event ns2 ESTABLISHED (1) [ OK ] # event ns2 ANNOUNCED (6) [ OK ] # event ns2 REMOVED (4) [ OK ] # event ns2 SUB_ESTABLISHED (5) [ OK ] # event ns2 SUB_CLOSED (3) [ OK ] # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # rm addr rx client [ OK ] # rm subflow server [ OK ] # 123 flush re-add # Info: Test file (size 4 KB) for client # Info: Test file (size 4 KB) for server # before flush [ OK ] # mptcp_info subflows=0:0 [ OK ] # join Rx [ OK ] # join Tx [ OK ] # add addr rx [ OK ] # add addr echo rx [ OK ] # rm addr rx client [ OK ] # rm subflow server [ OK ] # # TAP version 13 # 1..123 # ok 1 - mptcp_join: no JOIN # time=1148ms # ok 2 - mptcp_join: single subflow, limited by client # time=1117ms # ok 3 - mptcp_join: single subflow, limited by server # time=1103ms # ok 4 - mptcp_join: single subflow # time=1111ms # ok 5 - mptcp_join: multiple subflows # time=1091ms # ok 6 - mptcp_join: multiple subflows, limited by server # time=1110ms # ok 7 - mptcp_join: single subflow, dev # time=1081ms # ok 8 - mptcp_join: no MPC reuse with single endpoint # time=6156ms # ok 9 - mptcp_join: multi subflows, with failing subflow # time=6255ms # ok 10 - mptcp_join: multi subflows, with subflow timeout # time=6239ms # ok 11 - mptcp_join: multi subflows, fair usage on close # time=6426ms # ok 12 - mptcp_join: unused signal address # time=1229ms # ok 13 - mptcp_join: signal address # time=1252ms # ok 14 - mptcp_join: subflow and signal # time=1350ms # ok 15 - mptcp_join: subflow and signal together # time=1441ms # ok 16 - mptcp_join: multiple subflows and signal # time=1298ms # ok 17 - mptcp_join: signal addresses # time=1279ms # ok 18 - mptcp_join: signal invalid addresses # time=1296ms # ok 19 - mptcp_join: signal addresses race test # time=6310ms # ok 20 - mptcp_join: multiple flows, signal, link failure # time=7919ms # ok 21 - mptcp_join: multi flows, signal, bidi, link fail # time=9704ms # ok 22 - mptcp_join: backup subflow unused, link failure # time=12865ms # ok 23 - mptcp_join: backup flow used, multi links fail # time=12531ms # ok 24 - mptcp_join: backup flow used, bidi, link failure # time=16857ms # ok 25 - mptcp_join: signal address, ADD_ADDR timeout # time=6504ms # ok 26 - mptcp_join: signal address, ADD_ADDR6 timeout # time=6452ms # ok 27 - mptcp_join: signal addresses, ADD_ADDR timeout # time=23337ms # ok 28 - mptcp_join: invalid address, ADD_ADDR timeout # time=23308ms # ok 29 - mptcp_join: remove single subflow # time=6534ms # ok 30 - mptcp_join: remove multiple subflows # time=6362ms # ok 31 - mptcp_join: remove single address # time=6408ms # ok 32 - mptcp_join: remove subflow and signal # time=6577ms # ok 33 - mptcp_join: remove subflows and signal # time=23344ms # ok 34 - mptcp_join: remove addresses # time=23658ms # ok 35 - mptcp_join: remove invalid addresses # time=23994ms # ok 36 - mptcp_join: flush subflows and signal # time=6605ms # ok 37 - mptcp_join: flush subflows # time=6532ms # ok 38 - mptcp_join: flush addresses # time=6544ms # ok 39 - mptcp_join: flush invalid addresses # time=6437ms # ok 40 - mptcp_join: remove id 0 subflow # time=6340ms # ok 41 - mptcp_join: remove id 0 address # time=6441ms # ok 42 - mptcp_join: add single subflow # time=6299ms # ok 43 - mptcp_join: add signal address # time=6364ms # ok 44 - mptcp_join: add multiple subflows # time=6281ms # ok 45 - mptcp_join: add multiple subflows IPv6 # time=6281ms # ok 46 - mptcp_join: add multiple addresses IPv6 # time=6387ms # ok 47 - mptcp_join: single subflow IPv6 # time=6386ms # ok 48 - mptcp_join: unused signal address IPv6 # time=6430ms # ok 49 - mptcp_join: single address IPv6 # time=6438ms # ok 50 - mptcp_join: remove single address IPv6 # time=6464ms # ok 51 - mptcp_join: remove subflow and signal IPv6 # time=6377ms # ok 52 - mptcp_join: single subflow IPv4-mapped # time=1257ms # ok 53 - mptcp_join: signal address IPv4-mapped # time=1282ms # ok 54 - mptcp_join: single subflow v4-map-v6 # time=1416ms # ok 55 - mptcp_join: signal address v4-map-v6 # time=1523ms # ok 56 - mptcp_join: single subflow v6-map-v4 # time=1469ms # ok 57 - mptcp_join: signal address v6-map-v4 # time=1462ms # ok 58 - mptcp_join: no JOIN with diff families v4-v6 # time=1329ms # ok 59 - mptcp_join: no JOIN with diff families v4-v6-2 # time=1217ms # ok 60 - mptcp_join: no JOIN with diff families v6-v4 # time=1324ms # ok 61 - mptcp_join: IPv4 sockets do not use IPv6 addresses # time=6359ms # ok 62 - mptcp_join: simult IPv4 and IPv6 subflows # time=6393ms # ok 63 - mptcp_join: simult IPv4 and IPv6 subflows, fullmesh 1x1 # time=6430ms # ok 64 - mptcp_join: simult IPv4 and IPv6 subflows, fullmesh 2x2 # time=6318ms # ok 65 - mptcp_join: single subflow, backup # time=6473ms # ok 66 - mptcp_join: single address, backup # time=6567ms # ok 67 - mptcp_join: single address, switch to backup # time=6652ms # ok 68 - mptcp_join: single address with port, backup # time=6661ms # ok 69 - mptcp_join: mpc backup # time=6534ms # ok 70 - mptcp_join: mpc backup both sides # time=6592ms # ok 71 - mptcp_join: mpc switch to backup # time=6485ms # ok 72 - mptcp_join: mpc switch to backup both sides # time=6451ms # ok 73 - mptcp_join: signal address with port # time=1497ms # ok 74 - mptcp_join: subflow and signal with port # time=1541ms # ok 75 - mptcp_join: remove single address with port # time=6825ms # ok 76 - mptcp_join: remove subflow and signal with port # time=6598ms # ok 77 - mptcp_join: flush subflows and signal with port # time=6486ms # ok 78 - mptcp_join: multiple addresses with port # time=1626ms # ok 79 - mptcp_join: multiple addresses with ports # time=1562ms # ok 80 - mptcp_join: port-based signal endpoint must not accept mpc # time=770ms # ok 81 - mptcp_join: single subflow with syn cookies # time=1282ms # ok 82 - mptcp_join: multiple subflows with syn cookies # time=1377ms # ok 83 - mptcp_join: subflows limited by server w cookies # time=1372ms # ok 84 - mptcp_join: signal address with syn cookies # time=1494ms # ok 85 - mptcp_join: subflow and signal w cookies # time=1481ms # ok 86 - mptcp_join: subflows and signal w. cookies # time=1570ms # ok 87 - mptcp_join: checksum test 0 0 # time=1574ms # ok 88 - mptcp_join: checksum test 1 1 # time=1653ms # ok 89 - mptcp_join: checksum test 0 1 # time=1588ms # ok 90 - mptcp_join: checksum test 1 0 # time=1537ms # ok 91 - mptcp_join: single subflow allow join id0 ns1 # time=1400ms # ok 92 - mptcp_join: single subflow allow join id0 ns2 # time=1481ms # ok 93 - mptcp_join: signal address allow join id0 ns1 # time=1552ms # ok 94 - mptcp_join: signal address allow join id0 ns2 # time=1371ms # ok 95 - mptcp_join: subflow and address allow join id0 1 # time=1353ms # ok 96 - mptcp_join: subflow and address allow join id0 2 # time=1435ms # ok 97 - mptcp_join: fullmesh test 2x1 # time=6482ms # ok 98 - mptcp_join: fullmesh test 1x1 # time=6381ms # ok 99 - mptcp_join: fullmesh test 1x2 # time=6262ms # ok 100 - mptcp_join: fullmesh test 1x2, limited # time=6327ms # ok 101 - mptcp_join: set fullmesh flag test # time=6307ms # ok 102 - mptcp_join: set nofullmesh flag test # time=6339ms # ok 103 - mptcp_join: set backup,fullmesh flags test # time=6727ms # ok 104 - mptcp_join: set nobackup,nofullmesh flags test # time=6540ms # ok 105 - mptcp_join: fastclose test # IGNORE Flaky time=62406ms # ok 106 - mptcp_join: fastclose server test # time=1424ms # ok 107 - mptcp_join: Infinite map # IGNORE Flaky time=62668ms # ok 108 - mptcp_join: MP_FAIL MP_RST # time=11082ms # ok 109 - mptcp_join: userspace pm type prevents add_addr # time=1472ms # ok 110 - mptcp_join: userspace pm no echo w/o daemon # time=1371ms # ok 111 - mptcp_join: userspace pm type rejects join # time=1308ms # ok 112 - mptcp_join: userspace pm type does not send join # time=1216ms # ok 113 - mptcp_join: userspace pm type prevents mp_prio # time=6432ms # ok 114 - mptcp_join: userspace pm type prevents rm_addr # time=6484ms # ok 115 - mptcp_join: userspace pm add & remove address # time=6238ms # ok 116 - mptcp_join: userspace pm create destroy subflow # time=3662ms # ok 117 - mptcp_join: userspace pm create id 0 subflow # time=3390ms # ok 118 - mptcp_join: userspace pm remove initial subflow # time=3527ms # ok 119 - mptcp_join: userspace pm send RM_ADDR for ID 0 # time=4871ms # ok 120 - mptcp_join: implicit EP # time=1073ms # ok 121 - mptcp_join: delete and re-add # time=13045ms # ok 122 - mptcp_join: delete re-add signal # time=8476ms # ok 123 - mptcp_join: flush re-add # time=4949ms ok 1 selftests: net/mptcp: mptcp_join.sh make[1]: Leaving directory '/home/virtme/testing-11/tools/testing/selftests/net/mptcp' make: Leaving directory '/home/virtme/testing-11/tools/testing/selftests' xx__-> echo $? 0 xx__->