Testing: Tests multiple nodes

.gitlab-ci.yml

@@ -123,6 +123,11 @@ test:contracts.sh:
   script:
     - make -C test run-contracts.sh
 
+test:multinode.sh:
+  <<: *test_definition
+  script:
+    - make -C test run-multinode.sh
+
 ## Publishing (small) docker images with tezos binaries
 
 publish:docker:minimal:

src/node/main/node_run_command.ml

@@ -20,6 +20,8 @@ let genesis : State.Net.genesis = {
       "ProtoGenesisGenesisGenesisGenesisGenesisGenesk612im" ;
 }
 
+type error += Nonlocalhost_sandbox of P2p_types.addr
+
 let (//) = Filename.concat
 
 let store_dir data_dir = data_dir // "store"
@@ -74,43 +76,48 @@ let init_node ?sandbox (config : Node_config_file.t) =
             | Ok json ->
                 Lwt.return (Some (patch_context (Some json)))
   end >>= fun patch_context ->
+  (* TODO "WARN" when pow is below our expectation. *)
   begin
-    match sandbox with
+    match config.net.listen_addr with
-    | Some _ -> return None
     | None ->
-        Node_identity_file.read
+        lwt_log_notice "Not listening to P2P calls." >>= fun () ->
-          (config.data_dir //
+        return (None, None)
-           Node_identity_file.default_name) >>=? fun identity ->
+    | Some addr ->
-        lwt_log_notice
+        Node_config_file.resolve_listening_addrs addr >>= function
-          "Peer's global id: %a"
+        | [] ->
-          P2p.Peer_id.pp identity.peer_id >>= fun () ->
+            failwith "Cannot resolve P2P listening address: %S" addr
-        (* TODO "WARN" when pow is below our expectation. *)
+        | (addr, port) :: _ -> return (Some addr, Some port)
-        begin
+  end >>=? fun (listening_addr, listening_port) ->
-          match config.net.listen_addr with
+  begin
-          | None ->
+    match listening_addr, sandbox with
-              lwt_log_notice "Not listening to RPC calls." >>= fun () ->
+    | Some addr, Some _
-              return (None, None)
+      when Ipaddr.V6.(compare addr unspecified) = 0 ->
-          | Some addr ->
+        return None
-              Node_config_file.resolve_listening_addrs addr >>= function
+    | Some addr, Some _ when Ipaddr.V6.(compare addr localhost) != 0 ->
-              | [] ->
+        fail (Nonlocalhost_sandbox addr)
-                  failwith "Cannot resolve RPC listening address: %S" addr
+    | None, Some _ -> return None
-              | (addr, port) :: _ -> return (Some addr, Some port)
+    | _ ->
-        end >>=? fun (listening_addr, listening_port) ->
+        (Node_config_file.resolve_bootstrap_addrs
-        Node_config_file.resolve_bootstrap_addrs
+           config.net.bootstrap_peers) >>= fun trusted_points ->
-          config.net.bootstrap_peers >>= fun trusted_points ->
+          Node_identity_file.read
-        let p2p_config : P2p.config =
+            (config.data_dir //
-          { listening_addr ;
+             Node_identity_file.default_name) >>=? fun identity ->
-            listening_port ;
+          lwt_log_notice
-            trusted_points ;
+            "Peer's global id: %a"
-            peers_file =
+            P2p.Peer_id.pp identity.peer_id >>= fun () ->
-              (config.data_dir // "peers.json") ;
+          let p2p_config : P2p.config =
-            closed_network = config.net.closed ;
+            { listening_addr ;
-            identity ;
+              listening_port ;
-            proof_of_work_target =
+              trusted_points ;
-              Crypto_box.make_target config.net.expected_pow ;
+              peers_file =
-          }
+                (config.data_dir // "peers.json") ;
-        in
+              closed_network = config.net.closed ;
-        return (Some (p2p_config, config.net.limits))
+              identity ;
+              proof_of_work_target =
+                Crypto_box.make_target config.net.expected_pow ;
+            }
+          in
+          return (Some (p2p_config, config.net.limits))
   end >>=? fun p2p_config ->
   let node_config : Node.config = {
     genesis ;
@@ -186,7 +193,18 @@ let process sandbox verbosity args =
     | [_] -> Some Logging.Info
     | _ -> Some Logging.Debug in
   let run =
-    Node_shared_arg.read_and_patch_config_file args >>=? fun config ->
+    Node_shared_arg.read_and_patch_config_file
+      ~ignore_bootstrap_peers:(match sandbox with
+          | Some _ -> true
+          | None -> false)
+      args >>=? fun config ->
+    begin match sandbox with
+      | Some _ ->
+          if config.data_dir = Node_config_file.default_data_dir
+          then failwith "Cannot use default data directory while in sandbox mode"
+          else return ()
+      | None -> return ()
+    end >>=? fun () ->
     Lwt_utils.Lock_file.is_locked
       (lock_file config.data_dir) >>=? function
     | false ->
@@ -210,7 +228,8 @@ module Term = struct
   let sandbox =
     let open Cmdliner in
     let doc =
-      "Run the daemon in sandbox mode. P2P is disabled, and constants of \
+      "Run the daemon in sandbox mode. \
+       P2P to non-localhost addressses are disabled, and constants of \
        the economic protocol can be altered with an optional JSON file. \
        $(b,IMPORTANT): Using sandbox mode affects the node state and \
        subsequent runs of Tezos node must also use sandbox mode. \

src/node/main/node_shared_arg.ml

@@ -9,6 +9,7 @@
 
 open Cmdliner
 open P2p_types
+open Logging.Node.Main
 
 let (//) = Filename.concat
 
@@ -242,7 +243,7 @@ module Term = struct
 
 end
 
-let read_and_patch_config_file args =
+let read_and_patch_config_file ?(ignore_bootstrap_peers=false) args =
   begin
     if Sys.file_exists args.config_file then
       Node_config_file.read args.config_file
@@ -260,10 +261,12 @@ let read_and_patch_config_file args =
         cors_origins ; cors_headers ;
         log_output } = args in
   let bootstrap_peers =
-    if no_bootstrap_peers then
+    if no_bootstrap_peers || ignore_bootstrap_peers
-      peers
+    then peers
-    else
+    else begin
-      cfg.net.bootstrap_peers @ peers in
+      log_info "Ignoring bootstrap peers";
+      cfg.net.bootstrap_peers @ peers
+    end in
   return @@
   Node_config_file.update
     ?data_dir ?min_connections ?expected_connections ?max_connections

src/node/main/node_shared_arg.mli

@@ -37,7 +37,7 @@ module Term : sig
   val config_file: string option Cmdliner.Term.t
 end
 
-val read_and_patch_config_file: t -> Node_config_file.t tzresult Lwt.t
+val read_and_patch_config_file: ?ignore_bootstrap_peers:bool -> t -> Node_config_file.t tzresult Lwt.t
 
 module Manpage : sig
   val misc_section: string

test/Makefile

@@ -31,3 +31,6 @@ run-basic.sh:
 
 run-contracts.sh:
 	./test_contracts.sh
+
+run-multinode.sh:
+	./test_multinode.sh

test/test_basic.sh

@@ -4,6 +4,12 @@ set -e
 
 source test_utils.sh
 
+start_sandboxed_node
+sleep 3
+activate_alpha
+
+add_bootstrap_identities
+
 ${TZCLIENT} list known identities
 
 ${TZCLIENT} transfer 1000 from bootstrap1 to ${KEY1}

test/test_contracts.sh

@@ -4,6 +4,15 @@ set -e
 
 source test_utils.sh
 
+start_sandboxed_node
+sleep 3
+
+activate_alpha
+
+add_bootstrap_identities
+
+printf "\n\n"
+
 CONTRACT_PATH=contracts
 
 # FORMAT: assert_output contract_file storage input expected_result
@@ -201,3 +210,5 @@ assert_balance $BOOTSTRAP4_IDENTITY "4,000,100.00 ꜩ"
 account=tz1SuakBpFdG9b4twyfrSMqZzruxhpMeSrE5
 ${TZCLIENT} transfer 0.00 from bootstrap1 to default_account  -arg "\"$account\""
 assert_balance $account "100.00 ꜩ"
+
+printf "\nEnd of test\n"

test/test_multinode.sh

@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+
+source test_utils.sh
+
+export LWT_ASYNC_METHOD="none"
+
+node1_rpcs=3000
+node1_addr=[::1]:3001
+node2_rpcs=3002
+node2_addr=[::1]:3003
+node3_rpcs=3004
+node3_addr=[::1]:3005
+node4_rpcs=3006
+node4_addr=[::1]:3007
+
+CLIENT_1="$(make_client) -addr [::1] -port $node1_rpcs"
+CLIENT_2="$(make_client) -addr [::1] -port $node2_rpcs"
+CLIENT_3="$(make_client) -addr [::1] -port $node3_rpcs"
+CLIENT_4="$(make_client) -addr [::1] -port $node4_rpcs"
+
+
+
+assert_propagation_level() {
+    level=$1
+    printf "\n\nAsserting all nodes have reached level %s\n" "$level"
+    ${CLIENT_1} rpc call /blocks/head/proto/context/level \
+        | assert_in_output "\"level\": $level"
+    ${CLIENT_2} rpc call /blocks/head/proto/context/level \
+        | assert_in_output "\"level\": $level"
+    ${CLIENT_3} rpc call /blocks/head/proto/context/level \
+        | assert_in_output "\"level\": $level"
+    ${CLIENT_4} rpc call /blocks/head/proto/context/level \
+        | assert_in_output "\"level\": $level"
+}
+
+start_sandboxed_node --rpc-addr=[::1]:$node1_rpcs --net-addr=$node1_addr --peer=$node2_addr --no-bootstrap-peers
+start_sandboxed_node --rpc-addr=[::1]:$node2_rpcs --net-addr=$node2_addr --peer=$node1_addr --no-bootstrap-peers
+start_sandboxed_node --rpc-addr=[::1]:$node3_rpcs --net-addr=$node3_addr --peer=$node1_addr --no-bootstrap-peers
+start_sandboxed_node --rpc-addr=[::1]:$node4_rpcs --net-addr=$node4_addr --peer=$node1_addr --no-bootstrap-peers
+
+sleep 3
+
+printf "\n\n"
+
+activate_alpha [::1] $node1_rpcs
+
+sleep 3
+
+printf "\n\nAsserting protocol propagation\n"
+
+${CLIENT_1} rpc call /blocks/head/protocol \
+    | assert_in_output "ProtoALphaALphaALphaALphaALphaALphaALphaALphaDdp3zK"
+${CLIENT_2} rpc call /blocks/head/protocol \
+    | assert_in_output "ProtoALphaALphaALphaALphaALphaALphaALphaALphaDdp3zK"
+${CLIENT_3} rpc call /blocks/head/protocol \
+    | assert_in_output "ProtoALphaALphaALphaALphaALphaALphaALphaALphaDdp3zK"
+${CLIENT_4} rpc call /blocks/head/protocol \
+    | assert_in_output "ProtoALphaALphaALphaALphaALphaALphaALphaALphaDdp3zK"
+
+printf "\n\n"
+
+add_bootstrap_identities "${CLIENT_1}"
+add_bootstrap_identities "${CLIENT_2}"
+add_bootstrap_identities "${CLIENT_3}"
+add_bootstrap_identities "${CLIENT_4}"
+
+printf "\n\n"
+
+${CLIENT_1} mine for bootstrap1
+
+sleep 3
+
+assert_propagation_level 2
+${CLIENT_2} mine for bootstrap2
+
+sleep 3
+
+assert_propagation_level 3
+${CLIENT_3} mine for bootstrap3
+
+sleep 3
+
+assert_propagation_level 4
+${CLIENT_4} mine for bootstrap4
+
+sleep 3
+
+assert_propagation_level 5
+
+endorse_hash=$(${CLIENT_3} endorse for bootstrap3 | extract_operation_hash)
+
+transfer_hash=$(${CLIENT_4} transfer 500 from bootstrap1 to bootstrap3 | extract_operation_hash)
+
+sleep 3
+
+${CLIENT_4} mine for bootstrap4
+
+sleep 3
+
+assert_contains_operation() {
+    hash="$1"
+    printf "Asserting operations list contains '$hash'\n"
+    ${CLIENT_1} rpc call /blocks/head/operations with {} \
+        | assert_in_output $hash
+    ${CLIENT_2} rpc call /blocks/head/operations with {} \
+        | assert_in_output $hash
+    ${CLIENT_3} rpc call /blocks/head/operations with {} \
+        | assert_in_output $hash
+    ${CLIENT_4} rpc call /blocks/head/operations with {} \
+        | assert_in_output $hash
+}
+
+assert_contains_operation $endorse_hash
+assert_contains_operation $transfer_hash
+
+# printf "\nEnd of test"

test/test_utils.sh

@@ -1,58 +1,107 @@
-#!/bin/bash
+#!/usr/bin/env bash
-# Run this as a command in scripts to test if a contract produces the correct output
-# Write `source test-michelson.sh`
 
-DATA_DIR="$(mktemp -d -t tezos_node.XXXXXXXXXX)"
+# If the TZPATH environment variable is not set,
-CLIENT_DIR="$(mktemp -d -t tezos_client.XXXXXXXXXX)"
+# assume that we're in the test directory
+if [ -z "$TZPATH" ]; then
+    export TZPATH="../"
+fi
 
-TZCLIENT="../tezos-client -base-dir ${CLIENT_DIR}"
+# Global arrays for cleanup
-TZNODE=../tezos-node
+if [ -z "${CLEANUP_DIS}" ]; then
+    export CLEANUP_DIS=()
+fi
+
+if [ -z "${CLEANUP_PROCESSES}" ]; then
+    export CLEANUP_PROCESSES=()
+fi
 
 cleanup() {
-    [ -z "${TZNODE_PID}" ] || kill -9 ${TZNODE_PID} || true
+    for ps in "${CLEANUP_PROCESSES[@]}"; do
-    printf "\nNode's log:\n" > /dev/stderr
+        kill -9 $ps
-    cat $DATA_DIR/LOG > /dev/stderr
+    done
-    rm -fr ${DATA_DIR} ${CLIENT_DIR}
+    CLEANUP_PROCESSES=()
+    sleep 2
+
+    for node_dir in "${CLEANUP_DIS[@]}"; do
+        printf "\nNode's log:\n" > /dev/stderr
+        cat $node_dir/LOG > /dev/stderr
+        rm -rf $node_dir
+    done
+    CLEANUP_DIS=()
+
+    for client_dir in ${CLIENT_DIRS[@]}; do
+        rm -rf $client_dir
+    done
+    CLIENT_DIRS=()
 }
-trap cleanup EXIT QUIT INT SIGINT SIGKILL
+trap cleanup EXIT
 
-CUSTOM_PARAM="--sandbox sandbox.json"
+register_dir() {
-${TZNODE} run --data-dir "${DATA_DIR}" ${CUSTOM_PARAM} --rpc-addr "[::]:8732" > "$DATA_DIR"/LOG 2>&1 &
+    CLEANUP_DIS+=("$1")
-TZNODE_PID="$!"
+}
 
-echo "Created node, pid: ${TZNODE_PID}, log: $DATA_DIR/LOG" > /dev/stderr
+make_client () {
+    client_dir="$(mktemp -d -t tezos_client.XXXXXXXXXX)"
+    echo "${TZPATH}/tezos-client -base-dir ${client_dir}"
+}
+TZCLIENT=$(make_client)
+TZNODE="${TZPATH}/tezos-node"
 
-sleep 3
+CUSTOM_PARAM="--sandbox=${TZPATH}/test/sandbox.json"
+
+start_sandboxed_node() {
+    if [ "$#" == "0" ]; then
+        default_args=--rpc-addr=[::]:8732
+    else
+        default_args=""
+    fi
+
+    data_dir="$(mktemp -d -t tezos_node.XXXXXXXXXX)"
+    register_dir "$data_dir"
+    ${TZNODE} identity generate 0 --data-dir "${data_dir}"
+    ${TZNODE} config init --data-dir=${data_dir} --connections=2 --expected-pow=0.0
+    ${TZNODE} run --data-dir "${data_dir}" ${CUSTOM_PARAM} "$@" $default_args > "$data_dir"/LOG 2>&1 &
+    node_pid="$!"
+    CLEANUP_PROCESSES+=($node_pid)
+    echo "Created node, pid: ${node_pid}, log: $data_dir/LOG" > /dev/stderr
+}
+
+
+activate_alpha() {
+    addr=${1:-[::]}
+    port=${2:-8732}
+    ${TZCLIENT} -port $port -addr $addr \
+                -block genesis \
+                activate \
+                protocol ProtoALphaALphaALphaALphaALphaALphaALphaALphaDdp3zK \
+                with fitness 1 \
+                and key edskRhxswacLW6jF6ULavDdzwqnKJVS4UcDTNiCyiH6H8ZNnn2pmNviL7pRNz9kRxxaWQFzEQEcZExGHKbwmuaAcoMegj5T99z
+}
 
-${TZCLIENT} -block genesis \
-          activate \
-          protocol ProtoALphaALphaALphaALphaALphaALphaALphaALphaDdp3zK \
-          with fitness 1 \
-          and key edskRhxswacLW6jF6ULavDdzwqnKJVS4UcDTNiCyiH6H8ZNnn2pmNviL7pRNz9kRxxaWQFzEQEcZExGHKbwmuaAcoMegj5T99z
 
 
 run_contract_file () {
-	local contract=$1;
+    local contract=$1;
-	local storage=$2;
+    local storage=$2;
-	local input=$3;
+    local input=$3;
-	${TZCLIENT} run program "$contract" on storage "$storage" and input "$input";
+    ${TZCLIENT} run program "$contract" on storage "$storage" and input "$input";
 }
 
 assert_output () {
-	local contract=$1;
+    local contract=$1;
-	local input=$2;
+    local input=$2;
-	local storage=$3;
+    local storage=$3;
-	local expected=$4;
+    local expected=$4;
-  echo "Testing [$contract]"
+    echo "Testing [$contract]"
-	local output=$(run_contract_file "$contract" "$input" "$storage" | sed '1,/output/d' |
+    local output=$(run_contract_file "$contract" "$input" "$storage" | sed '1,/output/d' |
-				       sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' ||
+                       sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' ||
-				       { printf '\nTest failed with error at line %s\n' "$(caller)" > /dev/stderr;
+                       { printf '\nTest failed with error at line %s\n' "$(caller)" > /dev/stderr;
-				         exit 1; });
+                         exit 1; });
-	if [ "$expected" != "$output" ]; then
+    if [ "$expected" != "$output" ]; then
-		echo "Test at" `caller` failed > /dev/stderr;
+        echo "Test at" `caller` failed > /dev/stderr;
-		printf "Expected %s but got %s" "$expected" "$output" > /dev/stderr;
+        printf "Expected %s but got %s" "$expected" "$output" > /dev/stderr;
-		exit 1;
+        exit 1;
-	fi
+    fi
 }
 
 assert_balance () {
@@ -95,9 +144,11 @@ assert_in_output () {
     local MATCHING="$1"
     local INPUT=${2-/dev/stdin}
     if ! grep -q "${MATCHING}" ${INPUT}; then
-        printf "Failure on line %s. Expected to find %s in output." \
+        printf "\nFailure on line %s. Expected to find %s in output." \
                "$(caller)"  "${MATCHING}"
         exit 1
+    else
+        echo "[Assertion succeeded]"
     fi
 }
 
@@ -131,25 +182,49 @@ BOOTSTRAP1_IDENTITY=tz1KqTpEZ7Yob7QbPE4Hy4Wo8fHG8LhKxZSx
 BOOTSTRAP1_PUBLIC=edpkuBknW28nW72KG6RoHtYW7p12T6GKc7nAbwYX5m8Wd9sDVC9yav
 BOOTSTRAP1_SECRET=edskRuR1azSfboG86YPTyxrQgosh5zChf5bVDmptqLTb5EuXAm9rsnDYfTKhq7rDQujdn5WWzwUMeV3agaZ6J2vPQT58jJAJPi
 BOOTSTRAP2_IDENTITY=tz1gjaF81ZRRvdzjobyfVNsAeSC6PScjfQwN
+BOOTSTRAP2_PUBLIC=edpktzNbDAUjUk697W7gYg2CRuBQjyPxbEg8dLccYYwKSKvkPvjtV9
+BOOTSTRAP2_SECRET=edskRkJz4Rw2rM5NtabEWMbbg2bF4b1nfFajaqEuEk4SgU7eeDbym9gVQtBTbYo32WUg2zb5sNBkD1whRN7zX43V9bftBbtaKc
 BOOTSTRAP3_IDENTITY=tz1faswCTDciRzE4oJ9jn2Vm2dvjeyA9fUzU
+BOOTSTRAP3_PUBLIC=edpkuTXkJDGcFd5nh6VvMz8phXxU3Bi7h6hqgywNFi1vZTfQNnS1RV
+BOOTSTRAP3_SECRET=edskS3qsqsNgdjUqeMsVcEwBn8dkZ5iDRz6aF21KhcCtRiAkWBypUSbicccR4Vgqm9UdW2Vabuos6seezqgbXTrmcbLUG4rdAC
 BOOTSTRAP4_IDENTITY=tz1b7tUupMgCNw2cCLpKTkSD1NZzB5TkP2sv
+BOOTSTRAP4_PUBLIC=edpkuFrRoDSEbJYgxRtLx2ps82UdaYc1WwfS9sE11yhauZt5DgCHbU
+BOOTSTRAP4_SECRET=edskRg9qcPqaVQa6jXWNMU5p71tseSuR7NzozgqZ9URsVDi81wTyPJdFSBdeakobyHUi4Xgu61jgKRQvkhXrPmEdEUfiqfiJFL
 BOOTSTRAP5_IDENTITY=tz1ddb9NMYHZi5UzPdzTZMYQQZoMub195zgv
+BOOTSTRAP5_PUBLIC=edpkv8EUUH68jmo3f7Um5PezmfGrRF24gnfLpH3sVNwJnV5bVCxL2n
+BOOTSTRAP5_SECRET=edskS7rLN2Df3nbS1EYvwJbWo4umD7yPM1SUeX7gp1WhCVpMFXjcCyM58xs6xsnTsVqHQmJQ2RxoAjJGedWfvFmjQy6etA3dgZ
 
 KEY1=foo
 KEY2=bar
 
-${TZCLIENT} add identity bootstrap1 ${BOOTSTRAP1_IDENTITY}
+add_bootstrap_identities() {
-${TZCLIENT} add public key bootstrap1 ${BOOTSTRAP1_PUBLIC}
+    client=${1:-${TZCLIENT}}
-${TZCLIENT} add secret key bootstrap1 ${BOOTSTRAP1_SECRET}
+    ${client} add identity bootstrap1 ${BOOTSTRAP1_IDENTITY}
-${TZCLIENT} add identity bootstrap2 ${BOOTSTRAP2_IDENTITY}
+    ${client} add public key bootstrap1 ${BOOTSTRAP1_PUBLIC}
-${TZCLIENT} add identity bootstrap3 ${BOOTSTRAP3_IDENTITY}
+    ${client} add secret key bootstrap1 ${BOOTSTRAP1_SECRET}
-${TZCLIENT} add identity bootstrap4 ${BOOTSTRAP4_IDENTITY}
-${TZCLIENT} add identity bootstrap5 ${BOOTSTRAP5_IDENTITY}
 
-sleep 2
+    ${client} add identity bootstrap2 ${BOOTSTRAP2_IDENTITY}
+    ${client} add public key bootstrap2 ${BOOTSTRAP2_PUBLIC}
+    ${client} add secret key bootstrap2 ${BOOTSTRAP2_SECRET}
 
-${TZCLIENT} gen keys ${KEY1}
+    ${client} add identity bootstrap3 ${BOOTSTRAP3_IDENTITY}
-${TZCLIENT} gen keys ${KEY2}
+    ${client} add public key bootstrap3 ${BOOTSTRAP3_PUBLIC}
+    ${client} add secret key bootstrap3 ${BOOTSTRAP3_SECRET}
 
-# For ease of use outside of the script
+    ${client} add identity bootstrap4 ${BOOTSTRAP4_IDENTITY}
-alias client="${TZCLIENT}"
+    ${client} add public key bootstrap4 ${BOOTSTRAP4_PUBLIC}
+    ${client} add secret key bootstrap4 ${BOOTSTRAP4_SECRET}
+
+    ${client} add identity bootstrap5 ${BOOTSTRAP5_IDENTITY}
+    ${client} add public key bootstrap5 ${BOOTSTRAP5_PUBLIC}
+    ${client} add secret key bootstrap5 ${BOOTSTRAP5_SECRET}
+
+    sleep 2
+
+    ${client} gen keys ${KEY1}
+    ${client} gen keys ${KEY2}
+}
+
+extract_operation_hash() {
+    grep "Operation hash is" | grep -o "'.*'" | tr -d "'"
+}