AGENT-1579: Agent-based installer - add static networking support for ISO_NO_REGISTRY mode

agent/06_agent_create_cluster.sh

@@ -335,6 +335,88 @@ function automate_rendezvousIP_selection(){
     done
 }
 
+function hostname_format_for_node_type(){
+  case "$1" in
+    master) echo "$MASTER_HOSTNAME_FORMAT" ;;
+    worker) echo "$WORKER_HOSTNAME_FORMAT" ;;
+    arbiter) echo "$ARBITER_HOSTNAME_FORMAT" ;;
+  esac
+}
+
+# Configuration step of the 'static_ip' test case: drive the agent TUI to
+# assign a static IP and hostname to each node of the given type. The static
+# IPs are derived from staticIPForNode (see agent/common.sh).
+function test_case_do_static_ip(){
+  local node_type=$1
+  local node_count=$2
+
+  local hostname_format
+  hostname_format="$(hostname_format_for_node_type "$node_type")"
+
+  for (( n=0; n<node_count; n++ ))
+    do
+        name=${CLUSTER_NAME}_${node_type}_${n}
+        node_ip="$(staticIPForNode "$node_type" "$n")"
+        # shellcheck disable=SC2059
+        node_hostname="$(printf "$hostname_format" "$n")"
+
+        sudo virsh screenshot "$name" "${OCP_DIR}/${name}_console_screenshot_before_static_networking.ppm"
+
+        echo "Configuring static IP ${node_ip} and hostname ${node_hostname} on ${name}"
+        ./agent/e2e/agent-tui/test-static-ip.sh "$name" "$node_ip" "$node_hostname"
+
+        sudo virsh screenshot "$name" "${OCP_DIR}/${name}_console_screenshot_after_static_networking.ppm"
+        echo "Finished configuring static networking for $name"
+    done
+}
+
+# Verification step of the 'static_ip' test case: confirm each node of the given
+# type is reachable at the static IP it was assigned via the agent TUI, and that
+# its hostname was applied. Exits non-zero if any node fails verification.
+function test_case_verify_static_ip(){
+  local node_type=$1
+  local node_count=$2
+
+  local hostname_format
+  hostname_format="$(hostname_format_for_node_type "$node_type")"
+
+  for (( n=0; n<node_count; n++ ))
+    do
+        node_ip="$(staticIPForNode "$node_type" "$n")"
+        # shellcheck disable=SC2059
+        expected_hostname="$(printf "$hostname_format" "$n")"
+
+        echo "Verifying ${node_type}-${n} is reachable at static IP ${node_ip} with hostname ${expected_hostname}"
+
+        # The UI reports the console URL as soon as it is available, but the
+        # nodes are still rebooting and finalizing (applying the static IP via
+        # coreos-installer --copy-network) at that point, so the static IP is not
+        # immediately reachable. Retry until the node comes up before failing.
+        local actual_hostname=""
+        local retries=10   # up to ~5 min at 30s interval
+        local i
+        for (( i=1; i<=retries; i++ )); do
+            if actual_hostname=$(${SSH} "core@${node_ip}" hostname 2>/dev/null) && [[ -n "${actual_hostname}" ]]; then
+                break
+            fi
+            actual_hostname=""
+            echo "Waiting for ${node_type}-${n} to be reachable at ${node_ip} (attempt ${i}/${retries})"
+            sleep 30
+        done
+
+        if [[ -z "${actual_hostname}" ]]; then
+            echo "ERROR: static_ip test case failed - could not ssh to ${node_type}-${n} at static IP ${node_ip}"
+            return 1
+        fi
+        # Compare the short hostname (strip any domain suffix)
+        if [[ "${actual_hostname%%.*}" != "${expected_hostname}" ]]; then
+            echo "ERROR: static_ip test case failed - node at ${node_ip} has hostname '${actual_hostname}', expected '${expected_hostname}'"
+            return 1
+        fi
+        echo "Verified ${node_type}-${n}: reachable at ${node_ip} with hostname ${expected_hostname}"
+    done
+}
+
 function check_assisted_install_UI(){
   local rendezvousIP
   rendezvousIP=$(getRendezvousIP)
@@ -554,8 +636,11 @@ function mce_complete_deployment() {
   mce_apply_postinstallation_manifests "${mceManifests}"
 }
 
-function run_agent_test_cases() {
-  if [[ $AGENT_TEST_CASES =~ "bad_dns" ]]; then
+# Configuration step of the 'bad_dns' test case: wait for the nodes to reach the
+# agent-tui check screen (failing because of the bad DNS record injected into
+# agent-config.yaml), then fix the DNS on master-0 via agent-tui so the
+# installation can proceed.
+function test_case_do_bad_dns() {
     # wait 5 minutes for VMs to load and arrive at agent-tui check screen
     echo "Running test scenario: bad DNS record(s) in agent-config.yaml"
     echo "Waiting for 5 mins to arrive at agent-tui check screen"
@@ -577,6 +662,11 @@ function run_agent_test_cases() {
     sudo virsh screenshot "$name" "${OCP_DIR}/${name}_console_screenshot_after_dns_fix.ppm"
 
     echo "Finished fixing DNS through agent-tui"
+}
+
+function run_agent_test_cases() {
+  if [[ $AGENT_TEST_CASES =~ "bad_dns" ]]; then
+    test_case_do_bad_dns
   fi
 }
 
@@ -838,9 +928,20 @@ case "${AGENT_E2E_TEST_BOOT_MODE}" in
         echo "aarch64: CDROM media ejected from all VMs"
     fi
 
-    automate_rendezvousIP_selection master "$NUM_MASTERS"
-    automate_rendezvousIP_selection worker "$NUM_WORKERS"
-    automate_rendezvousIP_selection arbiter "$NUM_ARBITERS"
+    if [[ "${AGENT_TEST_CASES:-}" =~ "static_ip" ]]; then
+        # The static_ip test case configures IPv4 static addresses only.
+        if [[ "${IP_STACK}" != "v4" ]]; then
+            echo "ERROR: the 'static_ip' test case is only supported with IP_STACK=v4 (got '${IP_STACK}')"
+            exit 1
+        fi
+        test_case_do_static_ip master "$NUM_MASTERS"
+        test_case_do_static_ip worker "$NUM_WORKERS"
+        test_case_do_static_ip arbiter "$NUM_ARBITERS"
+    else
+        automate_rendezvousIP_selection master "$NUM_MASTERS"
+        automate_rendezvousIP_selection worker "$NUM_WORKERS"
+        automate_rendezvousIP_selection arbiter "$NUM_ARBITERS"
+    fi
 
     check_assisted_install_UI
 
@@ -852,6 +953,18 @@ case "${AGENT_E2E_TEST_BOOT_MODE}" in
     pushd agent/isobuilder/ui_driven_cluster_installation
     RENDEZVOUS_IP=$rendezvousIP OCP_DIR=$ocp_dir_abs_path INGRESS_VIPS=$INGRESS_VIPS API_VIPS=$API_VIPS go run main.go
     popd
+
+    # Verification step of the 'static_ip' test case: now that installation has
+    # completed, confirm each node came up with the static IP and hostname that
+    # were configured via the agent TUI.
+    if [[ "${AGENT_TEST_CASES:-}" =~ "static_ip" ]]; then
+        echo "Running test scenario: verify static IPs configured via agent-tui"
+        test_case_verify_static_ip master "$NUM_MASTERS"
+        test_case_verify_static_ip worker "$NUM_WORKERS"
+        test_case_verify_static_ip arbiter "$NUM_ARBITERS"
+        echo "Finished verifying static networking on all nodes"
+    fi
+
     exit 0
     ;;
 esac

agent/common.sh

@@ -99,7 +99,32 @@ if [ "${AGENT_E2E_TEST_BOOT_MODE}" == "ISO_NO_REGISTRY" ] ; then
     fi
 fi
 
+# Static IP assignment for the 'static_ip' agent TUI test case.
+# The rendezvous node (first master) is assigned .80 and each subsequent node
+# increments by one, ordered masters, then workers, then arbiters.
+export AGENT_STATIC_IP_PREFIX="192.168.111"
+export AGENT_STATIC_IP_BASE=80
+
+# staticIPForNode <node_type> <index> - echo the static IP assigned to a node.
+function staticIPForNode() {
+    local node_type=$1
+    local index=$2
+    local offset=0
+    case "${node_type}" in
+        master) offset=0 ;;
+        worker) offset=${NUM_MASTERS} ;;
+        arbiter) offset=$(( NUM_MASTERS + NUM_WORKERS )) ;;
+    esac
+    echo "${AGENT_STATIC_IP_PREFIX}.$(( AGENT_STATIC_IP_BASE + offset + index ))"
+}
+
 function getRendezvousIP() {
+    # For the 'static_ip' test case the rendezvous node is assigned a known
+    # static IP via the agent TUI, so return that directly.
+    if [[ "${AGENT_TEST_CASES:-}" =~ "static_ip" ]]; then
+        staticIPForNode master 0
+        return
+    fi
     if [[ "${NODES_PLATFORM}" == "baremetal" ]]; then
         echo "${AGENT_BAREMETAL_IPS%%,*}"
         return

agent/e2e/agent-tui/test-static-ip.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+#
+# test-static-ip.sh - configuration step of the 'static_ip' e2e test case.
+#
+# Drives the agent TUI (and nmtui within it) via "virsh send-key" to assign a
+# static IPv4 address and hostname to a single node, creating a NEW
+# NetworkManager connection so coreos-installer's --copy-network preserves the
+# static config into the installed system. The matching verification step runs
+# at the end of the install (see test_case_verify_static_ip in
+# agent/06_agent_create_cluster.sh).
+#
+# Example configuration applied to a node:
+#   ┌───────────────┬─────────────────┐
+#   │ Interface     │ enp2s0          │
+#   │ IP address    │ <node_ip>/24    │
+#   │ Gateway       │ 192.168.111.1   │
+#   │ DNS server    │ 192.168.111.1   │
+#   │ Hostname      │ <node_hostname> │
+#   │ Rendezvous IP │ <rendezvous IP> │
+#   └───────────────┴─────────────────┘
+#
+# Usage: test-static-ip.sh <node_name> <node_ip> <node_hostname>
+#
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../../" && pwd )"
+source "$SCRIPTDIR/common.sh"
+source "$SCRIPTDIR/agent/common.sh"
+source "$SCRIPTDIR/agent/e2e/agent-tui/utils.sh"
+
+set +x
+
+node_name=$1
+node_ip=$2
+node_hostname=$3
+
+# All interaction below is via "virsh send-key" into agent-tui and nmtui.
+
+# Phase 1: from the TUI first screen navigate to <Configure Network>.
+pressDown "Navigate to <Configure Network>" 3 "$node_name"
+pressEnter "Select '<Configure Network>'" "" "$node_name"
+
+# Phase 2: add a new Ethernet connection with a static IP in nmtui. A NEW
+# connection (not editing the existing one) gives a .nmconnection file with a
+# birth time after agent-tui start, which triggers coreos-installer --copy-network.
+pressEnter "Select 'Edit a connection'" "" "$node_name"
+
+# Tab from the connection list to the <Add> button.
+pressTab "Goto <Add> button" 1 "$node_name"
+pressEnter "Select '<Add>' button" "" "$node_name"
+
+# Connection type dialog: select Ethernet, then tab to <Create> and confirm.
+pressDown "Select Ethernet" 1 "$node_name"
+pressTab "Goto <Cancel> button" 1 "$node_name"
+pressTab "Goto <Create> button" 1 "$node_name"
+pressEnter "Create new Ethernet connection" "" "$node_name"
+
+# Leave default profile name, tab to the Device field and enter the interface.
+pressTab "Goto Device field" 1 "$node_name"
+pressKeys "Enter device name" "enp2s0" "$node_name"
+
+# Tab past ETHERNET/802.1X <Show> to the IPv4 CONFIGURATION mode.
+pressTab "Goto IPv4 CONFIGURATION mode" 3 "$node_name"
+pressEnter "Open IPv4 mode selector" "" "$node_name"
+# From Automatic, Manual is two DOWNs (Disabled, Automatic, Link-Local, Manual).
+pressDown "Select Manual" 2 "$node_name"
+pressEnter "Confirm Manual" "" "$node_name"
+
+# Expand <Show> to reveal the address fields.
+pressTab "Goto <Show>" 1 "$node_name"
+pressEnter "Expand IPv4 details" "" "$node_name"
+
+# Enter the static IP in the Addresses field.
+pressTab "Goto Addresses <Add...>" 1 "$node_name"
+pressEnter "Activate Addresses field" "" "$node_name"
+pressKeys "Enter static IP address" "${node_ip}/24" "$node_name"
+
+# Tab past <Remove>/<Add...> to the Gateway field.
+pressTab "Goto Gateway field" 3 "$node_name"
+pressKeys "Enter gateway" "192.168.111.1" "$node_name"
+
+# Enter the DNS server.
+pressTab "Goto DNS servers <Add...>" 1 "$node_name"
+pressEnter "Activate DNS servers field" "" "$node_name"
+pressKeys "Enter DNS server" "192.168.111.1" "$node_name"
+
+# Tab past the remaining DNS/search/routing/IPv6 options and checkboxes to <OK>
+# (14 stops: <Remove>, <Add...>, Search domains, Routing, 4 route/DNS checkboxes,
+# IPv6 <Automatic>/<Show>, 2 connect checkboxes, <Cancel>, <OK>).
+pressTab "Goto <OK> button" 14 "$node_name"
+pressEnter "Select '<OK>' button" "" "$node_name"
+
+# Phase 3: back at the connection list, go Back to the main menu and activate
+# the new static connection (NM auto-deactivates the old DHCP one).
+pressTab "Goto <Back> button" 4 "$node_name"
+pressEnter "Select '<Back>' button" "" "$node_name"
+pressDown "Select 'Activate a connection'" 1 "$node_name"
+pressEnter "Select 'Activate a connection' menu item" "" "$node_name"
+# Our new static connection is 2 DOWN (past the active '* Wired connection 2').
+pressDown "Select 'Ethernet connection 1'" 2 "$node_name"
+pressEnter "Activate new static connection" "" "$node_name"
+sleep 3
+
+# Phase 4: set the system hostname in nmtui.
+pressTab "Goto <Back> button" 2 "$node_name"
+pressEnter "Select '<Back>' button" "" "$node_name"
+pressDown "Select 'Set system hostname'" 1 "$node_name"
+pressEnter "Select 'Set system hostname' menu item" "" "$node_name"
+
+# Clear any existing hostname (End then Ctrl+U) and enter the new one.
+pressKey "Goto end of hostname field" KEY_END 1 "$node_name"
+echo "Clear hostname field"
+sudo virsh send-key "$node_name" KEY_LEFTCTRL KEY_U
+sleep 1
+pressKeys "Enter hostname" "$node_hostname" "$node_name"
+
+# Tab past <Cancel> to <OK> and confirm.
+pressTab "Goto <OK> button" 2 "$node_name"
+pressEnter "Confirm hostname" "" "$node_name"
+
+# Phase 5: exit nmtui back to the TUI first screen (Quit is 2 DOWN from hostname).
+pressDown "Select Quit" 2 "$node_name"
+pressEnter "Select 'Quit' menu item" "" "$node_name"
+pressEsc "Esc from network tree view" 2 "$node_name"
+sleep 3
+
+# Wait for TUI checks to update after network reconfiguration.
+sleep 10
+
+# Phase 6: save the rendezvous IP on the TUI first screen. All nodes enter the
+# same rendezvous IP; the rendezvous node recognizes it as its own.
+rendezvousIP=$(getRendezvousIP)
+pressKeys "Entering rendezvous IP address" "$rendezvousIP" "$node_name"
+# After typing, the first TAB is absorbed, so 2 TABs reach <Save rendezvous IP>.
+pressTab "Goto <Save rendezvous IP>" 2 "$node_name"
+pressEnter "" "" "$node_name"
+pressEnter "Save and Continue" "" "$node_name"

agent/e2e/agent-tui/utils.sh

@@ -35,7 +35,7 @@ function pressKey() {
     echo "$msg"
   fi	
 
-  local node_name=$4
+  local node_name=${4:-}
   for i in $(seq 1 "$numReps"); do
     _pressKey "$keyCode" "$node_name"
   done
@@ -46,19 +46,19 @@ function pressEnter() {
 }
 
 function pressTab() {
-  pressKey "$1" KEY_TAB "$2"
+  pressKey "$1" KEY_TAB "$2" "${3:-}"
 }
 
 function pressDown() {
   pressKey "$1" KEY_DOWN "$2" "$3"
 }
 
 function pressBackspace() {
-  pressKey "$1" KEY_BACKSPACE "$2"
+  pressKey "$1" KEY_BACKSPACE "$2" "${3:-}"
 }
 
 function pressEsc() {
-  pressKey "$1" KEY_ESC "$2"
+  pressKey "$1" KEY_ESC "$2" "${3:-}"
 }
 
 function pressKeys(){
@@ -85,6 +85,10 @@ function pressKeys(){
       c="DOT"
     elif [[ $c =~ [':'] ]]; then
       c="LEFTSHIFT KEY_SEMICOLON"
+    elif [[ $c =~ ['/'] ]]; then
+      c="SLASH"
+    elif [[ $c == '-' ]]; then
+      c="MINUS"
     fi
 
     local keyCode="KEY_"$c

config_example.sh

@@ -994,7 +994,6 @@ set -x
 # As the size of the ISO increases in future, increase the expected ISO size accordingly.
 # export AGENT_OVE_ISO_SIZE=40
 
-
 # Uncomment and set the following value to "true" to enable a test scenario
 # where the DNS is disabled on the hosts by setting its IP address to an incorrect value.
 # Uncomment and set the following value to one or more test cases below.
@@ -1011,6 +1010,19 @@ set -x
 # then the wait-for commands should timeout and fail.
 # This test case is only supported when IP_STACK=v4.
 #
+# 2. 'static_ip' test case:
+# Configures static networking on the nodes via the agent TUI (nmtui) instead of
+# using DHCP, then verifies at the end of the installation that each node came up
+# with the configured static IP and hostname. During the agent TUI automation a
+# new NetworkManager connection is created on each node with a static IPv4
+# address and hostname; static IPs are assigned starting at 192.168.111.80
+# (rendezvous node), incrementing by 1 for each additional node. After the
+# installation completes, the test ssh's to each node at its assigned static IP
+# and confirms the hostname matches, failing the deployment if any node is not
+# reachable at its static IP.
+# This test case requires AGENT_E2E_TEST_BOOT_MODE=ISO_NO_REGISTRY and is only
+# supported when IP_STACK=v4.
+#
 # export AGENT_TEST_CASES='bad_dns'
 
 # Uncomment the following line to deploy the cluster using the appliance model