Improve validation test (#2013)

.github/workflows/validate-1m-linux.sh

@@ -3,7 +3,7 @@
 # This test script joins Earth and pokes some stuff
 
 TEST_NETWORK=8056c2e21c000001
-RUN_LENGTH=10
+RUN_LENGTH=60
 TEST_FINISHED=false
 ZTO_VER=$(git describe --tags $(git rev-list --tags --max-count=1))
 ZTO_COMMIT=$(git rev-parse HEAD)
@@ -16,394 +16,358 @@ mkdir $TEST_DIR_PREFIX
 ################################################################################
 # Multi-node connectivity and performance test                                 #
 ################################################################################
+main() {
+	echo -e "\nRunning test for $RUN_LENGTH seconds"
+	NS1="ip netns exec ns1"
+	NS2="ip netns exec ns2"
 
-NS1="ip netns exec ns1"
-NS2="ip netns exec ns2"
+	ZT1="$NS1 ./zerotier-cli -p9996 -D$(pwd)/node1"
+	# Specify custom port on one node to ensure that feature works
+	ZT2="$NS2 ./zerotier-cli -p9997 -D$(pwd)/node2"
 
-ZT1="$NS1 ./zerotier-cli -D$(pwd)/node1"
-# Specify custom port on one node to ensure that feature works
-ZT2="$NS2 ./zerotier-cli -p9997 -D$(pwd)/node2"
+	echo -e "\nSetting up network namespaces..."
+	echo "Setting up ns1"
 
-echo -e "Setting up network namespaces..."
-echo "Setting up ns1"
+	ip netns add ns1
+	$NS1 ip link set dev lo up
+	ip link add veth0 type veth peer name veth1
+	ip link set veth1 netns ns1
+	ip addr add 192.168.0.1/24 dev veth0
+	ip link set dev veth0 up
+
+	$NS1 ip addr add 192.168.0.2/24 dev veth1
+	$NS1 ip link set dev veth1 up
+
+	# Add default route
+	$NS1 ip route add default via 192.168.0.1
+
+	iptables -t nat -A POSTROUTING -s 192.168.0.0/255.255.255.0 \
+		-o eth0 -j MASQUERADE
+	iptables -A FORWARD -i eth0 -o veth0 -j ACCEPT
+	iptables -A FORWARD -o eth0 -i veth0 -j ACCEPT
+
+	echo "Setting up ns2"
+	ip netns add ns2
+	$NS2 ip link set dev lo up
+	ip link add veth2 type veth peer name veth3
+	ip link set veth3 netns ns2
+	ip addr add 192.168.1.1/24 dev veth2
+	ip link set dev veth2 up
+
+	$NS2 ip addr add 192.168.1.2/24 dev veth3
+	$NS2 ip link set dev veth3 up
+	$NS2 ip route add default via 192.168.1.1
+
+	iptables -t nat -A POSTROUTING -s 192.168.1.0/255.255.255.0 \
+		-o eth0 -j MASQUERADE
+	iptables -A FORWARD -i eth0 -o veth2 -j ACCEPT
+	iptables -A FORWARD -o eth0 -i veth2 -j ACCEPT
+
+	# Allow forwarding
+	sysctl -w net.ipv4.ip_forward=1
+
+	echo -e "\nPing from host to namespaces"
+
+	ping -c 3 192.168.0.1
+	ping -c 3 192.168.1.1
 
-ip netns add ns1
-$NS1 ip link set dev lo up
-ip link add veth0 type veth peer name veth1
-ip link set veth1 netns ns1
-ip addr add 192.168.0.1/24 dev veth0
-ip link set dev veth0 up
+	echo -e "\nPing from namespace to host"
+
+	$NS1 ping -c 3 192.168.0.1
+	$NS1 ping -c 3 192.168.0.1
+	$NS2 ping -c 3 192.168.0.2
+	$NS2 ping -c 3 192.168.0.2
 
-$NS1 ip addr add 192.168.0.2/24 dev veth1
-$NS1 ip link set dev veth1 up
+	echo -e "\nPing from ns1 to ns2"
 
-# Add default route
-$NS1 ip route add default via 192.168.0.1
+	$NS1 ping -c 3 192.168.0.1
 
-iptables -t nat -A POSTROUTING -s 192.168.0.0/255.255.255.0 \
-      -o eth0 -j MASQUERADE
-iptables -A FORWARD -i eth0 -o veth0 -j ACCEPT
-iptables -A FORWARD -o eth0 -i veth0 -j ACCEPT
+	echo -e "\nPing from ns2 to ns1"
 
-echo "Setting up ns2"
-ip netns add ns2
-$NS2 ip link set dev lo up
-ip link add veth2 type veth peer name veth3
-ip link set veth3 netns ns2
-ip addr add 192.168.1.1/24 dev veth2
-ip link set dev veth2 up
+	$NS2 ping -c 3 192.168.0.1
 
-$NS2 ip addr add 192.168.1.2/24 dev veth3
-$NS2 ip link set dev veth3 up
-$NS2 ip route add default via 192.168.1.1
+	################################################################################
+	# Memory Leak Check                                                            #
+	################################################################################
 
-iptables -t nat -A POSTROUTING -s 192.168.1.0/255.255.255.0 \
-      -o eth0 -j MASQUERADE
-iptables -A FORWARD -i eth0 -o veth2 -j ACCEPT
-iptables -A FORWARD -o eth0 -i veth2 -j ACCEPT
+	FILENAME_MEMORY_LOG="$TEST_FILEPATH_PREFIX-memory.log"
 
-# Allow forwarding
-sysctl -w net.ipv4.ip_forward=1
+	echo -e "\nStarting a ZeroTier instance in each namespace..."
 
-echo -e "\nPing from host to namespaces"
+	time_test_start=$(date +%s)
 
-ping -c 3 192.168.0.1
-ping -c 3 192.168.1.1
+	# Spam the CLI as ZeroTier is starting
+	spam_cli 100
 
-echo -e "\nPing from namespace to host"
+	echo "Starting memory leak check"
+	$NS1 sudo valgrind --demangle=yes --exit-on-first-error=yes \
+		--error-exitcode=1 \
+		--xml=yes \
+		--xml-file=$FILENAME_MEMORY_LOG \
+		--leak-check=full \
+		./zerotier-one node1 -p9996 -U >>node_1.log 2>&1 &
 
-$NS1 ping -c 3 192.168.0.1
-$NS1 ping -c 3 192.168.0.1
-$NS2 ping -c 3 192.168.0.2
-$NS2 ping -c 3 192.168.0.2
+	# Second instance, not run in memory profiler
+	$NS2 sudo ./zerotier-one node2 -U -p9997 >>node_2.log 2>&1 &
 
-echo -e "\nPing from ns1 to ns2"
+	################################################################################
+	# Online Check                                                                 #
+	################################################################################
 
-$NS1 ping -c 3 192.168.0.1
+	echo "Waiting for ZeroTier to come online before attempting test..."
+	MAX_WAIT_SECS="${MAX_WAIT_SECS:-120}"
+	node1_online=false
+	node2_online=false
+	both_instances_online=false
+	time_zt_node1_start=$(date +%s)
+	time_zt_node2_start=$(date +%s)
 
-echo -e "\nPing from ns2 to ns1"
+	for ((s = 0; s <= MAX_WAIT_SECS; s++)); do
+		node1_online="$($ZT1 -j info | jq '.online' 2>/dev/null)"
+		node2_online="$($ZT2 -j info | jq '.online' 2>/dev/null)"
+		echo "Checking for online status: try #$s, node1:$node1_online, node2:$node2_online"
+		if [[ "$node1_online" == "true" ]]; then
+			time_zt_node1_online=$(date +%s)
+		fi
+		if [[ "$node2_online" == "true" ]]; then
+			time_zt_node2_online=$(date +%s)
+		fi
+		if [[ "$node2_online" == "true" && "$node1_online" == "true" ]]; then
+			both_instances_online=true
+			break
+		fi
+		sleep 1
+	done
+
+	echo -e "\n\nContents of ZeroTier home paths:"
+
+	ls -lga node1
+	tree node1
+	ls -lga node2
+	tree node2
+
+	echo -e "\n\nRunning ZeroTier processes:"
+	echo -e "\nNode 1:"
+	$NS1 ps aux | grep zerotier-one
+	echo -e "\nNode 2:"
+	$NS2 ps aux | grep zerotier-one
 
-$NS2 ping -c 3 192.168.0.1
+	echo -e "\n\nStatus of each instance:"
 
-################################################################################
-# Memory Leak Check                                                            #
-################################################################################
+	echo -e "\n\nNode 1:"
+	$ZT1 status
+	echo -e "\n\nNode 2:"
+	$ZT2 status
 
-FILENAME_MEMORY_LOG="$TEST_FILEPATH_PREFIX-memory.log"
+	if [[ "$both_instances_online" != "true" ]]; then
+		echo "One or more instances of ZeroTier failed to come online. Aborting test."
+		exit 1
+	fi
 
-echo -e "\nStarting a ZeroTier instance in each namespace..."
+	echo -e "\nJoining networks"
 
-time_test_start=`date +%s`
+	$ZT1 join $TEST_NETWORK
+	$ZT2 join $TEST_NETWORK
 
-# Spam the CLI as ZeroTier is starting
-spam_cli 100
+	sleep 10
 
-echo "Starting memory leak check"
-$NS1 sudo valgrind --demangle=yes --exit-on-first-error=yes \
-      --error-exitcode=1 \
-      --xml=yes \
-      --xml-file=$FILENAME_MEMORY_LOG \
-      --leak-check=full \
-      ./zerotier-one node1 -U >>node_1.log 2>&1 &
+	node1_ip4=$($ZT1 get $TEST_NETWORK ip4)
+	node2_ip4=$($ZT2 get $TEST_NETWORK ip4)
 
-# Second instance, not run in memory profiler
-$NS2 sudo ./zerotier-one node2 -U -p9997 >>node_2.log 2>&1 &
+	echo "node1_ip4=$node1_ip4"
+	echo "node2_ip4=$node2_ip4"
 
-################################################################################
-# Online Check                                                                 #
-################################################################################
+	echo -e "\nPinging each node"
 
-spam_cli()
-{
-      echo "Spamming CLI..."
-      # Rapidly spam the CLI with joins/leaves
-
-      MAX_TRIES="${$1:-10}"
-
-      for ((s=0; s<=MAX_TRIES; s++))
-      do
-            $ZT1 status
-            $ZT2 status
-            sleep 0.1
-      done
-
-      SPAM_TRIES=128
-
-      for ((s=0; s<=SPAM_TRIES; s++))
-      do
-            $ZT1 join $TEST_NETWORK
-      done
-
-      for ((s=0; s<=SPAM_TRIES; s++))
-      do
-            $ZT1 leave $TEST_NETWORK
-      done
-
-      for ((s=0; s<=SPAM_TRIES; s++))
-      do
-            $ZT1 leave $TEST_NETWORK
-            $ZT1 join $TEST_NETWORK
-      done
-}
+	PING12_FILENAME="$TEST_FILEPATH_PREFIX-ping-1-to-2.txt"
+	PING21_FILENAME="$TEST_FILEPATH_PREFIX-ping-2-to-1.txt"
 
-echo "Waiting for ZeroTier to come online before attempting test..."
-MAX_WAIT_SECS="${MAX_WAIT_SECS:-120}"
-node1_online=false
-node2_online=false
-both_instances_online=false
-time_zt_node1_start=`date +%s`
-time_zt_node2_start=`date +%s`
-
-for ((s=0; s<=MAX_WAIT_SECS; s++))
-do
-    node1_online="$($ZT1 -j info | jq '.online' 2>/dev/null)"
-    node2_online="$($ZT2 -j info | jq '.online' 2>/dev/null)"
-    echo "Checking for online status: try #$s, node1:$node1_online, node2:$node2_online"
-    if [[ "$node1_online" == "true" ]]
-    then
-        time_zt_node1_online=`date +%s`
-    fi
-    if [[ "$node2_online" == "true" ]]
-    then
-        time_zt_node2_online=`date +%s`
-    fi
-    if [[ "$node2_online" == "true" && "$node1_online" == "true" ]]
-    then
-        both_instances_online=true
-        break
-    fi
-    sleep 1
-done
-
-echo -e "\n\nContents of ZeroTier home paths:"
-
-ls -lga node1
-tree node1
-ls -lga node2
-tree node2
-
-echo -e "\n\nRunning ZeroTier processes:"
-echo -e "\nNode 1:"
-$NS1 ps aux | grep zerotier-one
-echo -e "\nNode 2:"
-$NS2 ps aux | grep zerotier-one
-
-echo -e "\n\nStatus of each instance:"
-
-echo -e "\n\nNode 1:"
-$ZT1 status
-echo -e "\n\nNode 2:"
-$ZT2 status
-
-if [[ "$both_instances_online" != "true" ]]
-then
-    echo "One or more instances of ZeroTier failed to come online. Aborting test."
-    exit 1
-fi
-
-echo -e "\nJoining networks"
-
-$ZT1 join $TEST_NETWORK
-$ZT2 join $TEST_NETWORK
-
-sleep 10
-
-node1_ip4=$($ZT1 get $TEST_NETWORK ip4)
-node2_ip4=$($ZT2 get $TEST_NETWORK ip4)
-
-echo "node1_ip4=$node1_ip4"
-echo "node2_ip4=$node2_ip4"
-
-echo -e "\nPinging each node"
-
-PING12_FILENAME="$TEST_FILEPATH_PREFIX-ping-1-to-2.txt"
-PING21_FILENAME="$TEST_FILEPATH_PREFIX-ping-2-to-1.txt"
-
-$NS1 ping -c 16 $node2_ip4 > $PING12_FILENAME
-$NS2 ping -c 16 $node1_ip4 > $PING21_FILENAME
-
-# Parse ping statistics
-ping_loss_percent_1_to_2="${ping_loss_percent_1_to_2:-100.0}"
-ping_loss_percent_2_to_1="${ping_loss_percent_2_to_1:-100.0}"
-
-ping_loss_percent_1_to_2=$(cat $PING12_FILENAME | \
-      grep "packet loss" | awk '{print $6}' | sed 's/%//')
-ping_loss_percent_2_to_1=$(cat $PING21_FILENAME | \
-      grep "packet loss" | awk '{print $6}' | sed 's/%//')
-
-# Normalize loss value
-ping_loss_percent_1_to_2=$(echo "scale=2; $ping_loss_percent_1_to_2/100.0" | bc)
-ping_loss_percent_2_to_1=$(echo "scale=2; $ping_loss_percent_2_to_1/100.0" | bc)
+	$NS1 ping -c 16 $node2_ip4 >$PING12_FILENAME
+	$NS2 ping -c 16 $node1_ip4 >$PING21_FILENAME
 
-################################################################################
-# CLI Check                                                                    #
-################################################################################
+	# Parse ping statistics
+	ping_loss_percent_1_to_2="${ping_loss_percent_1_to_2:-100.0}"
+	ping_loss_percent_2_to_1="${ping_loss_percent_2_to_1:-100.0}"
 
-echo "Testing basic CLI functionality..."
-
-spam_cli 10
-
-$ZT1 join $TEST_NETWORK
-
-$ZT1 -h
-$ZT1 -v
-$ZT1 status
-$ZT1 info
-$ZT1 listnetworks
-$ZT1 peers
-$ZT1 listpeers
-
-$ZT1 -j status
-$ZT1 -j info
-$ZT1 -j listnetworks
-$ZT1 -j peers
-$ZT1 -j listpeers
-
-$ZT1 dump
-
-$ZT1 get $TEST_NETWORK allowDNS
-$ZT1 get $TEST_NETWORK allowDefault
-$ZT1 get $TEST_NETWORK allowGlobal
-$ZT1 get $TEST_NETWORK allowManaged
-$ZT1 get $TEST_NETWORK bridge
-$ZT1 get $TEST_NETWORK broadcastEnabled
-$ZT1 get $TEST_NETWORK dhcp
-$ZT1 get $TEST_NETWORK id
-$ZT1 get $TEST_NETWORK mac
-$ZT1 get $TEST_NETWORK mtu
-$ZT1 get $TEST_NETWORK name
-$ZT1 get $TEST_NETWORK netconfRevision
-$ZT1 get $TEST_NETWORK nwid
-$ZT1 get $TEST_NETWORK portDeviceName
-$ZT1 get $TEST_NETWORK portError
-$ZT1 get $TEST_NETWORK status
-$ZT1 get $TEST_NETWORK type
-
-# Test an invalid command
-$ZT1 get $TEST_NETWORK derpderp
-
-# TODO: Validate JSON
+	ping_loss_percent_1_to_2=$(cat $PING12_FILENAME |
+		grep "packet loss" | awk '{print $6}' | sed 's/%//')
+	ping_loss_percent_2_to_1=$(cat $PING21_FILENAME |
+		grep "packet loss" | awk '{print $6}' | sed 's/%//')
 
-################################################################################
-# Performance Test                                                             #
-################################################################################
+	# Normalize loss value
+	ping_loss_percent_1_to_2=$(echo "scale=2; $ping_loss_percent_1_to_2/100.0" | bc)
+	ping_loss_percent_2_to_1=$(echo "scale=2; $ping_loss_percent_2_to_1/100.0" | bc)
 
-FILENAME_PERF_JSON="$TEST_FILEPATH_PREFIX-iperf.json"
+	################################################################################
+	# CLI Check                                                                    #
+	################################################################################
 
-echo -e "\nBeginning performance test:"
+	echo "Testing basic CLI functionality..."
 
-echo -e "\nStarting server:"
+	spam_cli 10
 
-echo "$NS1 iperf3 -s &"
-sleep 1
+	$ZT1 join $TEST_NETWORK
 
-echo -e "\nStarting client:"
-sleep 1
+	$ZT1 -h
+	$ZT1 -v
+	$ZT1 status
+	$ZT1 info
+	$ZT1 listnetworks
+	$ZT1 peers
+	$ZT1 listpeers
 
-echo "$NS2 iperf3 --json -c $node1_ip4 > $FILENAME_PERF_JSON"
+	$ZT1 -j status
+	$ZT1 -j info
+	$ZT1 -j listnetworks
+	$ZT1 -j peers
+	$ZT1 -j listpeers
 
-cat $FILENAME_PERF_JSON
+	$ZT1 dump
 
-################################################################################
-# Collect ZeroTier dump files                                                  #
-################################################################################
+	$ZT1 get $TEST_NETWORK allowDNS
+	$ZT1 get $TEST_NETWORK allowDefault
+	$ZT1 get $TEST_NETWORK allowGlobal
+	$ZT1 get $TEST_NETWORK allowManaged
+	$ZT1 get $TEST_NETWORK bridge
+	$ZT1 get $TEST_NETWORK broadcastEnabled
+	$ZT1 get $TEST_NETWORK dhcp
+	$ZT1 get $TEST_NETWORK id
+	$ZT1 get $TEST_NETWORK mac
+	$ZT1 get $TEST_NETWORK mtu
+	$ZT1 get $TEST_NETWORK name
+	$ZT1 get $TEST_NETWORK netconfRevision
+	$ZT1 get $TEST_NETWORK nwid
+	$ZT1 get $TEST_NETWORK portDeviceName
+	$ZT1 get $TEST_NETWORK portError
+	$ZT1 get $TEST_NETWORK status
+	$ZT1 get $TEST_NETWORK type
 
-echo -e "\nCollecting ZeroTier dump files"
+	# Test an invalid command
+	$ZT1 get $TEST_NETWORK derpderp
 
-node1_id=$($ZT1 -j status | jq -r .address)
-node2_id=$($ZT2 -j status | jq -r .address)
+	# TODO: Validate JSON
 
-$ZT1 dump
-mv zerotier_dump.txt "$TEST_FILEPATH_PREFIX-node-dump-$node1_id.txt"
+	################################################################################
+	# Performance Test                                                             #
+	################################################################################
 
-$ZT2 dump
-mv zerotier_dump.txt "$TEST_FILEPATH_PREFIX-node-dump-$node2_id.txt"
+	FILENAME_PERF_JSON="$TEST_FILEPATH_PREFIX-iperf.json"
 
-################################################################################
-# Let ZeroTier idle long enough for various timers                             #
-################################################################################
+	echo -e "\nBeginning performance test:"
 
-echo -e "\nIdling ZeroTier for $RUN_LENGTH seconds..."
-sleep $RUN_LENGTH
+	echo -e "\nStarting server:"
 
-echo -e "\nLeaving networks"
+	echo "$NS1 iperf3 -s &"
+	sleep 1
 
-$ZT1 leave $TEST_NETWORK
-$ZT2 leave $TEST_NETWORK
+	echo -e "\nStarting client:"
+	sleep 1
 
-sleep 5
+	echo "$NS2 iperf3 --json -c $node1_ip4 > $FILENAME_PERF_JSON"
 
-################################################################################
-# Stop test                                                                    #
-################################################################################
+	cat $FILENAME_PERF_JSON
 
-echo -e "\nStopping memory check..."
-sudo pkill -15 -f valgrind
-sleep 10
+	################################################################################
+	# Collect ZeroTier dump files                                                  #
+	################################################################################
 
-time_test_end=`date +%s`
+	echo -e "\nCollecting ZeroTier dump files"
 
-################################################################################
-# Rename ZeroTier stdout/stderr logs                                           #
-################################################################################
+	node1_id=$($ZT1 -j status | jq -r .address)
+	node2_id=$($ZT2 -j status | jq -r .address)
 
-mv node_1.log "$TEST_FILEPATH_PREFIX-node-log-$node1_id.txt"
-mv node_2.log "$TEST_FILEPATH_PREFIX-node-log-$node2_id.txt"
+	$ZT1 dump
+	mv zerotier_dump.txt "$TEST_FILEPATH_PREFIX-node-dump-$node1_id.txt"
 
-################################################################################
-# Generate report                                                              #
-################################################################################
+	$ZT2 dump
+	mv zerotier_dump.txt "$TEST_FILEPATH_PREFIX-node-dump-$node2_id.txt"
 
-cat $FILENAME_MEMORY_LOG
+	################################################################################
+	# Let ZeroTier idle long enough for various timers                             #
+	################################################################################
 
-DEFINITELY_LOST=$(xmlstarlet sel -t -v '/valgrindoutput/error/xwhat' \
-      $FILENAME_MEMORY_LOG | grep "definitely" | awk '{print $1;}')
-POSSIBLY_LOST=$(xmlstarlet sel -t -v '/valgrindoutput/error/xwhat' \
-      $FILENAME_MEMORY_LOG | grep "possibly" | awk '{print $1;}')
+	echo -e "\nIdling ZeroTier for $RUN_LENGTH seconds..."
+	sleep $RUN_LENGTH
 
-################################################################################
-# Generate coverage report artifact and summary                                #
-################################################################################
+	echo -e "\nLeaving networks"
 
-FILENAME_COVERAGE_JSON="$TEST_FILEPATH_PREFIX-coverage.json"
-FILENAME_COVERAGE_HTML="$TEST_FILEPATH_PREFIX-coverage.html"
+	$ZT1 leave $TEST_NETWORK
+	$ZT2 leave $TEST_NETWORK
 
-echo -e "\nGenerating coverage test report..."
+	sleep 5
 
-gcovr -r . --exclude ext --json-summary $FILENAME_COVERAGE_JSON \
-      --html > $FILENAME_COVERAGE_HTML
+	################################################################################
+	# Stop test                                                                    #
+	################################################################################
 
-cat $FILENAME_COVERAGE_JSON
+	echo -e "\nStopping memory check..."
+	sudo pkill -15 -f valgrind
+	sleep 10
 
-COVERAGE_LINE_COVERED=$(cat $FILENAME_COVERAGE_JSON | jq .line_covered)
-COVERAGE_LINE_TOTAL=$(cat $FILENAME_COVERAGE_JSON | jq .line_total)
-COVERAGE_LINE_PERCENT=$(cat $FILENAME_COVERAGE_JSON | jq .line_percent)
+	time_test_end=$(date +%s)
 
-COVERAGE_LINE_COVERED="${COVERAGE_LINE_COVERED:-0}"
-COVERAGE_LINE_TOTAL="${COVERAGE_LINE_TOTAL:-0}"
-COVERAGE_LINE_PERCENT="${COVERAGE_LINE_PERCENT:-0}"
+	################################################################################
+	# Rename ZeroTier stdout/stderr logs                                           #
+	################################################################################
 
-################################################################################
-# Default values                                                               #
-################################################################################
+	mv node_1.log "$TEST_FILEPATH_PREFIX-node-log-$node1_id.txt"
+	mv node_2.log "$TEST_FILEPATH_PREFIX-node-log-$node2_id.txt"
 
-DEFINITELY_LOST="${DEFINITELY_LOST:-0}"
-POSSIBLY_LOST="${POSSIBLY_LOST:-0}"
+	################################################################################
+	# Generate report                                                              #
+	################################################################################
 
-################################################################################
-# Summarize and emit json for trend reporting                                  #
-################################################################################
+	cat $FILENAME_MEMORY_LOG
+
+	DEFINITELY_LOST=$(xmlstarlet sel -t -v '/valgrindoutput/error/xwhat' \
+		$FILENAME_MEMORY_LOG | grep "definitely" | awk '{print $1;}')
+	POSSIBLY_LOST=$(xmlstarlet sel -t -v '/valgrindoutput/error/xwhat' \
+		$FILENAME_MEMORY_LOG | grep "possibly" | awk '{print $1;}')
+
+	################################################################################
+	# Generate coverage report artifact and summary                                #
+	################################################################################
+
+	FILENAME_COVERAGE_JSON="$TEST_FILEPATH_PREFIX-coverage.json"
+	FILENAME_COVERAGE_HTML="$TEST_FILEPATH_PREFIX-coverage.html"
+
+	echo -e "\nGenerating coverage test report..."
+
+	gcovr -r . --exclude ext --json-summary $FILENAME_COVERAGE_JSON \
+		--html >$FILENAME_COVERAGE_HTML
+
+	cat $FILENAME_COVERAGE_JSON
 
-FILENAME_SUMMARY="$TEST_FILEPATH_PREFIX-summary.json"
+	COVERAGE_LINE_COVERED=$(cat $FILENAME_COVERAGE_JSON | jq .line_covered)
+	COVERAGE_LINE_TOTAL=$(cat $FILENAME_COVERAGE_JSON | jq .line_total)
+	COVERAGE_LINE_PERCENT=$(cat $FILENAME_COVERAGE_JSON | jq .line_percent)
 
-time_length_test=$((time_test_end-time_test_start))
-time_length_zt_node1_online=$((time_zt_node1_online-time_zt_start))
-time_length_zt_node2_online=$((time_zt_node2_online-time_zt_start))
-#time_length_zt_join=$((time_zt_join_end-time_zt_join_start))
-#time_length_zt_leave=$((time_zt_leave_end-time_zt_leave_start))
-#time_length_zt_can_still_ping=$((time_zt_can_still_ping-time_zt_leave_start))
+	COVERAGE_LINE_COVERED="${COVERAGE_LINE_COVERED:-0}"
+	COVERAGE_LINE_TOTAL="${COVERAGE_LINE_TOTAL:-0}"
+	COVERAGE_LINE_PERCENT="${COVERAGE_LINE_PERCENT:-0}"
 
-summary=$(cat <<EOF
+	################################################################################
+	# Default values                                                               #
+	################################################################################
+
+	DEFINITELY_LOST="${DEFINITELY_LOST:-0}"
+	POSSIBLY_LOST="${POSSIBLY_LOST:-0}"
+
+	################################################################################
+	# Summarize and emit json for trend reporting                                  #
+	################################################################################
+
+	FILENAME_SUMMARY="$TEST_FILEPATH_PREFIX-summary.json"
+
+	time_length_test=$((time_test_end - time_test_start))
+	time_length_zt_node1_online=$((time_zt_node1_online - time_zt_start))
+	time_length_zt_node2_online=$((time_zt_node2_online - time_zt_start))
+	#time_length_zt_join=$((time_zt_join_end-time_zt_join_start))
+	#time_length_zt_leave=$((time_zt_leave_end-time_zt_leave_start))
+	#time_length_zt_can_still_ping=$((time_zt_can_still_ping-time_zt_leave_start))
+
+	summary=$(
+		cat <<EOF
 {
   "version":"$ZTO_VER",
   "commit":"$ZTO_COMMIT",
@@ -429,9 +393,42 @@ summary=$(cat <<EOF
   "mean_perf_netns": $POSSIBLY_LOST
 }
 EOF
-)
+	)
 
-echo $summary > $FILENAME_SUMMARY
-cat $FILENAME_SUMMARY
+	echo $summary >$FILENAME_SUMMARY
+	cat $FILENAME_SUMMARY
+}
+
+################################################################################
+# CLI Check                                                                    #
+################################################################################
+
+spam_cli() {
+	echo "Spamming CLI..."
+	# Rapidly spam the CLI with joins/leaves
+
+	MAX_TRIES="${1:-10}"
+
+	for ((s = 0; s <= MAX_TRIES; s++)); do
+		$ZT1 status
+		$ZT2 status
+		sleep 0.1
+	done
+
+	SPAM_TRIES=128
+
+	for ((s = 0; s <= SPAM_TRIES; s++)); do
+		$ZT1 join $TEST_NETWORK
+	done
+
+	for ((s = 0; s <= SPAM_TRIES; s++)); do
+		$ZT1 leave $TEST_NETWORK
+	done
+
+	for ((s = 0; s <= SPAM_TRIES; s++)); do
+		$ZT1 leave $TEST_NETWORK
+		$ZT1 join $TEST_NETWORK
+	done
+}
 
-"$@"
+main "$@"