Tests: increase timeouts in case the CI is too slow

test/test_multinode.sh

@@ -32,8 +32,9 @@ assert_propagation_level() {
     level=$1
     printf "\n\nAsserting all nodes have reached level %s\n" "$level"
     for client in "${client_instances[@]}"; do
-        $client rpc call /blocks/head/proto/context/level \
-            | assert_in_output "\"level\": $level"
+        ( $client rpc call /blocks/head/proto/context/level \
+              | assert_in_output "\"level\": $level" ) \
+            || exit 2
     done
 }
 
@@ -46,39 +47,53 @@ done
 
 printf "\n\n"
 
+retry() {
+    local timeout=$1
+    local attempts=$2
+    shift 2
+    sleep $timeout
+    while ! ( "$@" ) ; do
+        echo
+        echo "Will retry after $timeout seconds..."
+        echo
+        sleep $timeout
+        attempts=$(($attempts-1))
+        if [ "$attempts" -eq 0 ] ; then
+            echo
+            echo "Failed after too many retries" 1>&2
+            exit 1
+        fi
+    done
+}
+
 $client1 mine for bootstrap1
-sleep 5
-assert_propagation_level 2
+retry 2 15 assert_propagation_level 2
 
 $client2 mine for bootstrap2
-sleep 5
-assert_propagation_level 3
+retry 2 15 assert_propagation_level 3
 
 $client3 mine for bootstrap3
-sleep 5
-assert_propagation_level 4
+retry 2 15 assert_propagation_level 4
 
 $client4 mine for bootstrap4
-sleep 5
-assert_propagation_level 5
+retry 2 15 assert_propagation_level 5
 
 endorse_hash=$($client3 endorse for bootstrap3 | extract_operation_hash)
 transfer_hash=$($client4 transfer 500 from bootstrap1 to bootstrap3 | extract_operation_hash)
-sleep 5
 
-$client4 mine for bootstrap4
-sleep 5
+retry 2 15 $client4 mine for bootstrap4
 
 assert_contains_operation() {
     hash="$1"
     printf "Asserting operations list contains '$hash'\n"
     for client in "${client_instances[@]}"; do
-        $client rpc call /blocks/head/operations with {} \
-            | assert_in_output $hash
+        ( $client rpc call /blocks/head/operations with {} \
+              | assert_in_output $hash ) \
+            || exit 2
     done
 }
 
-assert_contains_operation $endorse_hash
+retry 2 15 assert_contains_operation $endorse_hash
 assert_contains_operation $transfer_hash
 
 echo