test: work around CI infrastructure failures

The CircleCI nested virt often fails to boot the VM. This patch catches
the infrastructure failures and retries up to 5 times.

Signed-off-by: David Scott <[email protected]>

Author: djs55

Makefile

@@ -189,10 +189,10 @@ test/vmlinuz test/initrd.gz:
 	@cd test; ./tinycore.sh
 
 test: $(TARGET) test/vmlinuz test/initrd.gz
-	@(cd test && ./test_linux.exp)
+	@(cd test && ./retry-infra-failure 5 ./test_linux.exp)
 
 test-qcow: $(TARGET) test/vmlinuz test/initrd.gz
-	@(cd test && ./test_linux_qcow.exp)
+	@(cd test && ./retry-infra-failure 5 ./test_linux_qcow.exp)
 
 
 ## ----------- ##

test/retry-infra-failure

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+retries=$1
+command=$2
+
+ret=0
+for((i=0;i<$retries;i++))
+do
+	$command
+	ret=$?
+	# Use exit code 2 to mean infra failure, need to retry.
+	if [ "$ret" != 2 ]; then
+		exit $ret
+	fi
+done
+
+echo "Persistent failure after $retries tries."
+exit $ret

test/test_linux.exp

@@ -15,6 +15,7 @@ set timeout 20
 
 expect {
   timeout {puts "FAIL boot"; exec kill -9 $pid; exit 1}
+  "Kernel panic - not syncing: IO-APIC + timer doesn't work" {puts "INFRA nested virtualization failed"; exec kill -9 $pid; exit 2}
   "\r\ntc@box:~$ "
 }
 send "sudo mount /dev/sda1 /mnt/sda1\r\n"

test/test_linux_qcow.exp

@@ -22,6 +22,7 @@ set timeout 20
 
 expect {
   timeout {puts "FAIL boot"; exec kill -9 $pid; exit 1}
+  "Kernel panic - not syncing: IO-APIC + timer doesn't work" {puts "INFRA nested virtualization failed"; exec kill -9 $pid; exit 2}
   "\r\ntc@box:~$ "
 }
 send "echo 'This is a block device' > /dev/sda\r\n"