feat(nightly): add QEMU integration tests for baremetal-sci_usi

.github/actions/test/integration/build/action.yml

@@ -4,6 +4,10 @@ inputs:
   image_tag:
     description: "Image tag to use for the build"
     required: true
+  uki_path:
+    description: "Optional local path to a pre-downloaded UKI file. If set, skip GHCR pull."
+    required: false
+    default: ""
 runs:
   using: "composite"
   steps:
@@ -22,19 +26,47 @@ runs:
         echo "DISK_SIZE=$DISK_SIZE" >> $GITHUB_ENV
 
     - name: Pull ORAS artifacts
+      if: ${{ inputs.uki_path == '' }}
       shell: bash
       run: |
-        LAYERS=$(oras manifest fetch "$REPO:$TAG" | jq -r '.layers[]')
-        UKI_SHA=$(echo $LAYERS | jq -r 'select(.mediaType=="application/io.gardenlinux.uki") | .digest')
-        TAR_SHA=$(echo "$LAYERS" | jq -r 'select(.mediaType=="application/io.gardenlinux.image.archive.format.tar")' | jq -rs 'sort_by(.size) | .[0].digest')
+        # oras manifest fetch in v1.2.2 has content negotiation issues with GHCR.
+        # Use curl with explicit Accept header to fetch the image manifest reliably.
+        ANON_TOKEN=$(curl -s "https://ghcr.io/token?service=ghcr.io&scope=repository:gardenlinux/gardenlinux-sci:pull" | jq -r '.token')
+        MANIFEST=$(curl -s \
+          -H "Authorization: Bearer $ANON_TOKEN" \
+          -H "Accept: application/vnd.oci.image.manifest.v1+json" \
+          "https://ghcr.io/v2/gardenlinux/gardenlinux-sci/manifests/$TAG")
+        echo "Manifest layers:"
+        echo "$MANIFEST" | jq -r '.layers[] | .mediaType' | head -5
+        UKI_SHA=$(echo "$MANIFEST" | jq -r '.layers[] | select(.mediaType=="application/io.gardenlinux.uki") | .digest')
+        echo "UKI digest: $UKI_SHA"
 
         oras blob fetch "$REPO@$UKI_SHA" -o "$RUNNER_TEMP/$TAG.uki"
-        oras blob fetch "$REPO@$TAR_SHA" -o "$RUNNER_TEMP/$TAG.tar" # wasteful, we only systemd-boot
-
-    - name: Extract systemd-boot
+        UKI_SIZE=$(stat -c%s "$RUNNER_TEMP/$TAG.uki")
+        echo "UKI size: $UKI_SIZE bytes"
+        if [ "$UKI_SIZE" -eq 0 ]; then
+          echo "::error::UKI blob is empty (digest: $UKI_SHA). The OCI artifact was built with an empty UKI — check the upstream build for version=today package issues."
+          exit 1
+        fi
+
+    - name: Use local UKI artifact
+      if: ${{ inputs.uki_path != '' }}
+      shell: bash
+      run: |
+        cp "${{ inputs.uki_path }}" "$RUNNER_TEMP/$TAG.uki"
+        UKI_SIZE=$(stat -c%s "$RUNNER_TEMP/$TAG.uki")
+        echo "Local UKI size: $UKI_SIZE bytes"
+        if [ "$UKI_SIZE" -eq 0 ]; then
+          echo "::error::Local UKI file is empty: ${{ inputs.uki_path }}"
+          exit 1
+        fi
+
+    - name: Generate SSH key
       shell: bash
       run: |
-        tar -xf "$RUNNER_TEMP/$TAG.tar" --no-same-owner --strip-components=5 -C "$RUNNER_TEMP" usr/lib/systemd/boot/efi/systemd-bootx64.efi
+        ssh-keygen -t ed25519 -f "/opt/ssh_host_ed25519_key" -N "" -C "user" -q
+        chmod 600 /opt/ssh_host_ed25519_key
+        echo "SSH_KEY=$(cat /opt/ssh_host_ed25519_key.pub)" >> $GITHUB_ENV
 
     - name: Set up a raw disk image
       shell: bash
@@ -57,63 +89,89 @@ runs:
         ESP_PART="${LOOPDEV}p1"
         echo "ESP_PART=$ESP_PART" >> $GITHUB_ENV
         VAR_PART="${LOOPDEV}p2"
+        echo "VAR_PART=$VAR_PART" >> $GITHUB_ENV
 
         sudo mkfs.vfat -F32 "$ESP_PART"
-        sudo mkfs.ext4 "$VAR_PART" -L VAR
+        sudo mkfs.ext4 -L VAR "$VAR_PART"
 
     - name: Populate the ESP partition
       shell: bash
       run: |
         mkdir -p "$RUNNER_TEMP/esp"
         sudo mount "$ESP_PART" "$RUNNER_TEMP/esp"
 
+        # Place the UKI directly as BOOTX64.EFI so OVMF loads it as an EFI binary.
+        # Systemd-boot's 'linux' entry type cannot load a UKI (it expects a bzImage);
+        # placing the UKI in EFI/Linux/ lets systemd-boot auto-discover it as a Type 2
+        # entry, but OVMF can boot the UKI directly without a bootloader.
         sudo mkdir -p "$RUNNER_TEMP/esp/EFI/BOOT"
-        sudo cp "$RUNNER_TEMP/systemd-bootx64.efi" "$RUNNER_TEMP/esp/EFI/BOOT/BOOTX64.EFI"
-        sudo cp "$RUNNER_TEMP/$TAG.uki" "$RUNNER_TEMP/esp/EFI/BOOT/gardenlinux.efi"
-
-        sudo mkdir -p "$RUNNER_TEMP/esp/loader/entries"
-
-        cat <<EOF | sudo tee "$RUNNER_TEMP/esp/loader/loader.conf" > /dev/null
-        default      gardenlinux
-        timeout      3
-        console-mode max
-        EOF
-        cat <<EOF | sudo tee "$RUNNER_TEMP/esp/loader/entries/gardenlinux.conf" > /dev/null
-        title   Garden Linux (UKI)
-        linux   /EFI/BOOT/gardenlinux.efi
-        options systemd.gpt_auto=0 audit=0 console=ttyS0 ignition.firstboot=1 ignition.platform.id=qemu
-        EOF
+        sudo cp "$RUNNER_TEMP/$TAG.uki" "$RUNNER_TEMP/esp/EFI/BOOT/BOOTX64.EFI"
 
         sudo umount "$RUNNER_TEMP/esp"
-        sudo losetup -d "$LOOPDEV"
 
-    - name: Convert to QCOW2
+    - name: Populate the VAR partition
       shell: bash
       run: |
-        qemu-img convert -O qcow2 "$DISK_IMG" "/opt/${TAG}.qcow2"
+        # Pre-populate the VAR partition so the UKI boots directly into the
+        # production system without running the firstboot/persist flow.
+        #
+        # The _usi initrd mounts VAR at /sysroot/var and then overlays /etc from
+        # var/etc.overlay. We pre-create the overlay dirs and write the SSH
+        # authorized_keys and systemd-networkd config so the system has network
+        # and SSH access on first boot.
+        mkdir -p "$RUNNER_TEMP/var"
+        sudo mount "$VAR_PART" "$RUNNER_TEMP/var"
+
+        # Overlay dirs required by sysroot-etc.mount
+        sudo mkdir -p "$RUNNER_TEMP/var/etc.overlay"
+        sudo mkdir -p "$RUNNER_TEMP/var/etc.overlay.workdir"
+
+        # Dirs required by sysroot-home.mount and sysroot-opt.mount bind mounts
+        sudo mkdir -p "$RUNNER_TEMP/var/home"
+        sudo mkdir -p "$RUNNER_TEMP/var/opt"
+
+        # SSH: allow root login with the CI key.
+        # We replace sshd_config in the overlay (it shadows the lowerdir copy).
+        # Mirror the base sshd_config from features/ssh/file.include but flip
+        # PermitRootLogin to yes and add AuthorizedKeysFile for the CI key.
+        sudo mkdir -p "$RUNNER_TEMP/var/etc.overlay/ssh"
+        echo "$SSH_KEY" | sudo tee "$RUNNER_TEMP/var/etc.overlay/ssh/authorized_keys" > /dev/null
+        sudo chmod 600 "$RUNNER_TEMP/var/etc.overlay/ssh/authorized_keys"
+        printf 'PermitRootLogin yes\nX11Forwarding no\nStrictModes yes\nIgnoreRhosts yes\nHostbasedAuthentication no\nLogLevel VERBOSE\nUsePAM yes\nPrintMotd no\nAcceptEnv LANG\nSubsystem sftp /usr/lib/openssh/sftp-server -f AUTHPRIV -l INFO\nClientAliveInterval 600\nClientAliveCountMax 0\nAuthenticationMethods publickey\nAuthorizedKeysFile /etc/ssh/authorized_keys\nHostKey /etc/ssh/ssh_host_ed25519_key\nHostKey /etc/ssh/ssh_host_rsa_key\nInclude /etc/ssh/sshd_config.d/*.conf\n' | sudo tee "$RUNNER_TEMP/var/etc.overlay/ssh/sshd_config" > /dev/null
+
+        # systemd-networkd: bring up DHCP on all Ethernet interfaces
+        sudo mkdir -p "$RUNNER_TEMP/var/etc.overlay/systemd/network"
+        printf '[Match]\nName=e*\n\n[Network]\nDHCP=yes\n' | sudo tee "$RUNNER_TEMP/var/etc.overlay/systemd/network/80-dhcp.network" > /dev/null
+
+        # Enable systemd-networkd, networkd-wait-online, and ssh via wants symlinks
+        # (systemctl doesn't work outside the running system)
+        sudo mkdir -p "$RUNNER_TEMP/var/etc.overlay/systemd/system/multi-user.target.wants"
+        sudo mkdir -p "$RUNNER_TEMP/var/etc.overlay/systemd/system/network-online.target.wants"
+        sudo ln -sf /usr/lib/systemd/system/systemd-networkd.service \
+            "$RUNNER_TEMP/var/etc.overlay/systemd/system/multi-user.target.wants/systemd-networkd.service"
+        sudo ln -sf /usr/lib/systemd/system/systemd-networkd-wait-online.service \
+            "$RUNNER_TEMP/var/etc.overlay/systemd/system/network-online.target.wants/systemd-networkd-wait-online.service"
+        sudo ln -sf /usr/lib/systemd/system/ssh.service \
+            "$RUNNER_TEMP/var/etc.overlay/systemd/system/multi-user.target.wants/ssh.service"
+
+        # Mask OVS services: ovsdb-server fails in CI (no persistent OVS DB)
+        # and ovs-vswitchd then blocks multi-user.target for 5 minutes.
+        # Masking (symlink to /dev/null) prevents them from starting.
+        sudo ln -sf /dev/null "$RUNNER_TEMP/var/etc.overlay/systemd/system/ovsdb-server.service"
+        sudo ln -sf /dev/null "$RUNNER_TEMP/var/etc.overlay/systemd/system/ovs-vswitchd.service"
+        sudo ln -sf /dev/null "$RUNNER_TEMP/var/etc.overlay/systemd/system/openvswitch-switch.service"
+
+        # Hosts file for HV-to-HV communication
+        # Use printf to avoid literal tabs which break YAML parsing
+        printf '127.0.0.1\tlocalhost\n127.0.1.1\tgarden\n::1\tlocalhost ip6-localhost ip6-loopback\nff02::1\tip6-allnodes\nff02::2\tip6-allrouters\n192.168.122.2 hv1\n192.168.122.3 hv2\n' | sudo tee "$RUNNER_TEMP/var/etc.overlay/hosts" > /dev/null
+
+        sudo umount "$RUNNER_TEMP/var"
+        sudo losetup -d "$LOOPDEV"
 
-    - name: Generate ignition
+    - name: Convert to QCOW2
       shell: bash
       run: |
-        ssh-keygen -t ed25519 -f "/opt/ssh_host_ed25519_key" -N "" -C "user" -q
-        chmod 600 /opt/ssh_host_ed25519_key
-        SSH_KEY=$(cat /opt/ssh_host_ed25519_key.pub)
-        echo "SSH_KEY=$SSH_KEY" >> $GITHUB_ENV
-        cp ./.github/actions/test/integration/build/dev-user-butane.yaml ./.github/actions/test/integration/build/dev-user-butane-hv1.yaml
-        yq '.storage.files += load("./.github/actions/test/integration/build/root-hints.yaml")' ./.github/actions/test/integration/build/dev-user-butane.yaml > ./.github/actions/test/integration/build/dev-user-butane-hv2.yaml
-
-        sed -i "s|SSH_KEY_GOES_HERE|$SSH_KEY|g" ./.github/actions/test/integration/build/dev-user-butane-hv1.yaml
-        sed -i "s|SSH_KEY_GOES_HERE|$SSH_KEY|g" ./.github/actions/test/integration/build/dev-user-butane-hv2.yaml
-
-        PASSWORD=$(openssl passwd "password")
-        echo "PASSWORD=$PASSWORD" >> $GITHUB_ENV
-        sed -i "s|PASSWORD_GOES_HERE|$PASSWORD|g" ./.github/actions/test/integration/build/dev-user-butane-hv1.yaml
-        sed -i "s|PASSWORD_GOES_HERE|$PASSWORD|g" ./.github/actions/test/integration/build/dev-user-butane-hv2.yaml
-        sed -i "s|TAG_GOES_HERE|${{ inputs.image_tag }}|g" ./.github/actions/test/integration/build/dev-user-butane-hv1.yaml
-        sed -i "s|TAG_GOES_HERE|${{ inputs.image_tag }}|g" ./.github/actions/test/integration/build/dev-user-butane-hv2.yaml
-
-        butane --pretty --strict ./.github/actions/test/integration/build/dev-user-butane-hv1.yaml > "/opt/${TAG}-hv1.ign"
-        butane --pretty --strict ./.github/actions/test/integration/build/dev-user-butane-hv2.yaml > "/opt/${TAG}-hv2.ign"
+        qemu-img convert -O qcow2 "$DISK_IMG" "/opt/${TAG}.qcow2"
 
     - name: Download ubuntu cloud image
       shell: bash
@@ -128,6 +186,8 @@ runs:
     - name: Generate ubuntu cloud-init
       shell: bash
       run: |
+        PASSWORD=$(openssl passwd "password")
+        echo "PASSWORD=$PASSWORD" >> $GITHUB_ENV
         sed -i "s|SSH_KEY_GOES_HERE|$SSH_KEY|g" ./.github/actions/test/integration/build/user-data
         sed -i "s|PASSWORD_GOES_HERE|$PASSWORD|g" ./.github/actions/test/integration/build/user-data
         genisoimage -output /opt/ubuntu-cloud-init-ds.iso -volid cidata -joliet -rock ./.github/actions/test/integration/build/meta-data ./.github/actions/test/integration/build/user-data

.github/actions/test/integration/build/dev-user-butane.yaml

@@ -52,6 +52,54 @@ storage:
         OCI_REPO=ghcr.io/gardenlinux/gardenlinux-sci
         # point to a custom tag to download for the persist step
         OCI_TAG=TAG_GOES_HERE
+  # Hook called by persist (chrooted into the new sysroot) after the UKI is
+  # downloaded and before systemd-repart partitions the disk.
+  # The chroot runs against the overlay-mounted sysroot: writes to /etc/ go
+  # to /sysroot/var/etc.overlay/ which is copied to the VAR partition by
+  # systemd-repart and applied as the /etc overlay upper layer on second boot.
+  - path: /opt/persist/network_up.sh
+    mode: 0755
+    contents:
+      inline: |
+        #!/usr/bin/env bash
+        set -Eeuo pipefail
+
+        # 1. Drop a systemd-networkd config so the second boot gets DHCP on all
+        #    Ethernet interfaces (virtio-net shows up as e*).
+        mkdir -p /etc/systemd/network
+        printf '[Match]\nName=e*\n\n[Network]\nDHCP=yes\n' \
+            > /etc/systemd/network/80-dhcp.network
+        echo "Wrote /etc/systemd/network/80-dhcp.network"
+
+        # 2. Enable systemd-networkd and ssh for the second boot by creating
+        #    the wants symlinks directly (systemctl may not work in chroot).
+        mkdir -p /etc/systemd/system/multi-user.target.wants
+        mkdir -p /etc/systemd/system/network-online.target.wants
+        ln -sf /usr/lib/systemd/system/systemd-networkd.service \
+            /etc/systemd/system/multi-user.target.wants/systemd-networkd.service || true
+        ln -sf /usr/lib/systemd/system/systemd-networkd-wait-online.service \
+            /etc/systemd/system/network-online.target.wants/systemd-networkd-wait-online.service || true
+        ln -sf /usr/lib/systemd/system/ssh.service \
+            /etc/systemd/system/multi-user.target.wants/ssh.service || true
+
+        # 3. Allow root SSH login with the authorized key on the second boot.
+        #    The first-boot ignition config wrote the key to /etc/ssh/authorized_keys
+        #    on the host (first-boot) system. We read it from outside the chroot
+        #    via /proc/1/root (the initrd's real root) and write it into the sysroot.
+        mkdir -p /etc/ssh
+        printf 'PermitRootLogin yes\nAuthorizedKeysFile /etc/ssh/authorized_keys\nProtocol 2\nAuthenticationMethods publickey\nSubsystem sftp /usr/lib/openssh/sftp-server\n' \
+            > /etc/ssh/sshd_config
+        # /proc is bind-mounted into the chroot by the kernel; /proc/1/root is
+        # the initrd root from which we can read the first-boot authorized_keys.
+        if [ -f /proc/1/root/etc/ssh/authorized_keys ]; then
+            cp /proc/1/root/etc/ssh/authorized_keys /etc/ssh/authorized_keys
+            chmod 600 /etc/ssh/authorized_keys
+            echo "Copied authorized_keys from first-boot into second-boot sysroot"
+        else
+            echo "WARNING: authorized_keys not found at /proc/1/root/etc/ssh/authorized_keys"
+        fi
+
+        echo "network_up.sh complete"
   # Enable unlimited core dumps for all systemd services
   - path: /etc/systemd/system.conf.d/10-core.conf
     mode: 0644
@@ -86,3 +134,5 @@ systemd:
       mask: true
     - name: libvirtd-tcp.socket
       mask: true
+    - name: ssh.service
+      enabled: true

.github/actions/test/integration/collect-logs/action.yml

@@ -13,6 +13,10 @@ runs:
         sudo cp /var/log/libvirt/qemu/HV1.log "libvirt-HV1.log" || true
         sudo cp /var/log/libvirt/qemu/HV2.log "libvirt-HV2.log" || true
 
+        # Serial console output (OVMF + kernel boot messages)
+        sudo cp /var/log/HV1.log "serial-HV1.log" || true
+        sudo cp /var/log/HV2.log "serial-HV2.log" || true
+
     - name: Obtain logs from hypervisors
       shell: bash
       run: |

.github/actions/test/integration/dependencies/action.yml

@@ -16,7 +16,8 @@ runs:
           libvirt-clients \
           virtinst \
           bridge-utils \
-          ovmf
+          ovmf \
+          python3-virt-firmware
 
     # Required for storing the ignition file in /var/lib/libvirt/images
     - name: Disable libvirt qemu security driver

.github/actions/test/integration/setup/action.yml

@@ -12,11 +12,28 @@ runs:
         sudo virsh net-start default
         sudo virsh net-autostart default
 
-    - name: Copy ignition file
+    - name: Create OVMF VARS with boot entry for HV disk
       shell: bash
       run: |
-        sudo cp "/opt/${TAG}-hv1.ign" /var/lib/libvirt/images/HV1.ign
-        sudo cp "/opt/${TAG}-hv2.ign" /var/lib/libvirt/images/HV2.ign
+        sudo mkdir -p /var/lib/libvirt/qemu/nvram
+
+        # Inject a File(\EFI\BOOT\BOOTX64.EFI) boot entry into VARS so OVMF
+        # finds it before the device-level Sata() entry created by bootindex=1.
+        # OVMF tries BootOrder entries in order; our entry becomes Boot0000 and
+        # goes first, so it finds the UKI at \EFI\BOOT\BOOTX64.EFI on the ESP.
+        #
+        # No extra kernel args in optional data: the UKI's embedded .cmdline is
+        # used as-is, and the VAR partition is pre-populated by the build step
+        # so the system boots directly into the production system without
+        # running the firstboot/persist flow.
+        virt-fw-vars \
+          --input /usr/share/OVMF/OVMF_VARS_4M.fd \
+          --output /tmp/VARS-hv.fd \
+          --append-boot-filepath '\EFI\BOOT\BOOTX64.EFI'
+        echo "VARS created successfully with boot entry"
+
+        sudo cp /tmp/VARS-hv.fd /var/lib/libvirt/qemu/nvram/HV1_VARS.fd
+        sudo cp /tmp/VARS-hv.fd /var/lib/libvirt/qemu/nvram/HV2_VARS.fd
 
     - name: Create HyperVisor 1
       shell: bash
@@ -46,7 +63,7 @@ runs:
         KEY=/opt/ssh_host_ed25519_key
         SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
         USER="root"
-        MAX_ITER=30
+        MAX_ITER=90
 
         setup_hv() {
           local IP="$1"
@@ -83,17 +100,23 @@ runs:
             virsh net-define /opt/virbr-vm.xml
             virsh net-start default
             virsh net-autostart default
+
+            # Create log directory for cloud-hypervisor virtlogd
+            mkdir -p /var/log/libvirt/ch
         EOF
         }
 
+        # The libvirt network assigns static DHCP reservations by MAC, so each
+        # HV always has a fixed IP (HV1 -> .2, HV2 -> .3). We SSH directly to
+        # that IP instead of polling `virsh domifaddr`: the lease table is
+        # cleared when the first boot reboots into the production image, so
+        # domifaddr is unreliable while the second boot re-acquires its lease.
         wait_for_ssh() {
           local VM_NAME="$1"
+          local IP="$2"
 
           for ((i=1; i<=MAX_ITER; i++)); do
-            local IP=$(sudo virsh domifaddr "$VM_NAME" | awk '/ipv4/ {print $4}' | cut -d'/' -f1)
-            if [ -z "$IP" ]; then
-              echo "No IP address found for $VM_NAME. Retrying..."
-            elif ssh $SSH_OPTS "$USER@$IP" 'exit' 2>/dev/null; then
+            if ssh $SSH_OPTS "$USER@$IP" 'exit' 2>/dev/null; then
               echo "$VM_NAME is up at IP: $IP"
               setup_hv "$IP"
               break
@@ -103,10 +126,10 @@ runs:
               sudo cat "/var/log/$VM_NAME.log"
               exit 1
             fi
-            echo "Waiting for $VM_NAME to respond to SSH... ($i/$MAX_ITER)"
+            echo "Waiting for $VM_NAME to respond to SSH at $IP... ($i/$MAX_ITER)"
             sleep 10
           done
         }
 
-        wait_for_ssh HV1
-        wait_for_ssh HV2
+        wait_for_ssh HV1 192.168.122.2
+        wait_for_ssh HV2 192.168.122.3