Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
* `ExternalValueSource` implementations for `RecvChain` and `Segments`.
* Producer batch completion without notification and segmented payload
assembly and extraction without flattening.
* Support register, memory, continue, single-step, interrupt, and software-breakpoint GDB debugging for Windows ARM64 guests.

### Changed
* `Sandbox` is the primary initialized sandbox type. `MultiUseSandbox` remains
Expand Down
15 changes: 11 additions & 4 deletions docs/how-to-debug-a-hyperlight-guest.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,22 @@ to start listening for a gdb connection.
## Supported features

The Hyperlight `gdb` feature enables guest debugging to:
- stop at an entry point breakpoint which is automatically set by Hyperlight
- add and remove HW breakpoints (maximum 4 set breakpoints at a time)
- stop before the first vCPU run
- add and remove SW breakpoints
- read and write registers
- read and write addresses
- step/continue
- continue
- single-step
- get code offset from target
- stop when a crash occurs and only allow read access to the guest memory and registers

On x86_64, gdb can also add up to four hardware breakpoints.
Windows ARM64 uses four-byte HVC software breakpoints and software single-step.
ARM64 stepping supports linear instructions, direct and conditional branches,
compare and test branches, and `BR`, `BLR`, and `RET`. Other control-flow and
exception instructions return an explicit error. Hardware breakpoints and
watchpoints are not advertised on Windows ARM64.

## Expected behavior

Below is a list describing some cases of expected behavior from a gdb debug
Expand Down Expand Up @@ -170,7 +177,7 @@ involved in the gdb debugging of a Hyperlight guest running inside a **KVM** or
| │ | create_gdb_thread | | │
| │ |◄─────────────────────────────────────────┌─┐ vcpu stopped ┌─┐ │
| attach │ ┌─┐ │ │◄──────────────────────────────┴─┘ │
┌─┐───────────────────────┼────────►│ │ │ │ entrypoint breakpoint | │
┌─┐───────────────────────┼────────►│ │ │ │ initial debug stop | │
│ │ attach response │ │ │ │ │ | │
│ │◄──────────────────────┼─────────│ │ │ │ | │
│ │ │ │ │ │ │ | │
Expand Down
3 changes: 2 additions & 1 deletion src/hyperlight_host/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,8 @@ fn main() -> Result<()> {
// Essentially the kvm and mshv3 features are ignored on windows as long as you use #[cfg(kvm)] and not #[cfg(feature = "kvm")].
// You should never use #[cfg(feature = "kvm")] or #[cfg(feature = "mshv3")] in the codebase.
cfg_aliases::cfg_aliases! {
gdb: { all(feature = "gdb", debug_assertions, target_arch = "x86_64") },
gdb_platform: { any(target_arch = "x86_64", all(target_arch = "aarch64", target_os = "windows")) },
gdb: { all(feature = "gdb", debug_assertions, gdb_platform) },
kvm: { all(feature = "kvm", target_os = "linux") },
mshv3: { all(feature = "mshv3", target_os = "linux") },
hvf: { all(feature = "hvf", target_os = "macos") },
Expand Down
74 changes: 73 additions & 1 deletion src/hyperlight_host/examples/guest-debugging/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,12 @@ fn main() -> hyperlight_host::Result<()> {
.host_function("Sleep5Secs", sleep_5_secs)
.build()?;

// Call guest function
#[cfg(target_arch = "x86_64")]
multi_use_sandbox_dbg
.call::<()>("UseSSE2Registers", ())
.unwrap();
#[cfg(target_arch = "aarch64")]
multi_use_sandbox_dbg.call::<()>("NoOp", ()).unwrap();

let message =
"Hello, World! I am executing inside of a VM with debugger attached :)\n".to_string();
Expand Down Expand Up @@ -291,6 +293,76 @@ mod tests {
}

#[test]
#[cfg(target_arch = "aarch64")]
#[serial]
fn test_gdb_continue_over_sw_breakpoint() {
let (out_file_path, cmd_file_path, manifest_dir) = gdb_test_paths("gdb-aarch64-continue");
let cmd = format!(
"file {manifest_dir}/../tests/rust_guests/bin/debug/simpleguest
target remote :8080
set pagination off
set logging file {out_file_path}
set logging enabled on
break simpleguest::no_op
commands 1
continue
end
break simpleguest::print_output
commands 2
echo Continued over ARM64 software breakpoint\\n
set logging enabled off
detach
quit
end
continue
"
);
#[cfg(windows)]
let cmd = format!("set osabi none\n{cmd}");
let checker =
|contents: String| contents.contains("Continued over ARM64 software breakpoint");
let result = run_guest_and_gdb(&cmd_file_path, &out_file_path, &cmd, checker);

cleanup(&out_file_path, &cmd_file_path);
assert!(result.is_ok(), "{}", result.unwrap_err());
}

#[test]
#[cfg(target_arch = "aarch64")]
#[serial]
fn test_gdb_single_step() {
let (out_file_path, cmd_file_path, manifest_dir) = gdb_test_paths("gdb-aarch64-step");
let cmd = format!(
"file {manifest_dir}/../tests/rust_guests/bin/debug/simpleguest
target remote :8080
set pagination off
set logging file {out_file_path}
set logging enabled on
break simpleguest::no_op
stepi
echo Stepped over ARM64 linear instruction\\n
continue
stepi
echo Stepped over ARM64 RET\\n
set logging enabled off
detach
quit
"
);
#[cfg(windows)]
let cmd = format!("set osabi none\n{cmd}");
let checker = |contents: String| {
contents.contains("Stepped over ARM64 linear instruction")
&& contents.contains("Stepped over ARM64 RET")
};
let result = run_guest_and_gdb(&cmd_file_path, &out_file_path, &cmd, checker);

cleanup(&out_file_path, &cmd_file_path);
assert!(result.is_ok(), "{}", result.unwrap_err());
}

#[test]
#[cfg(target_arch = "x86_64")]
#[serial]
fn test_gdb_sse_check() {
let (out_file_path, cmd_file_path, manifest_dir) = gdb_test_paths("gdb-sse");
Expand Down
97 changes: 9 additions & 88 deletions src/hyperlight_host/src/hypervisor/gdb/arch.rs
Original file line number Diff line number Diff line change
@@ -1,91 +1,12 @@
// SPDX-License-Identifier: Apache-2.0
// Copyright 2025 The Hyperlight Authors.

//! This file contains architecture specific code for the x86_64

use super::{DebugError, DebuggableVm, VcpuStopReason};
use crate::hypervisor::regs::CommonRegisters;
use crate::hypervisor::virtual_machine::RegisterError;

/// Errors that can occur when determining the vCPU stop reason
#[derive(Debug, thiserror::Error)]
pub enum VcpuStopReasonError {
#[error("Failed to get registers: {0}")]
GetRegs(#[from] RegisterError),
#[error("Failed to remove hardware breakpoint: {0}")]
RemoveHwBreakpoint(#[from] DebugError),
}

// Described in Table 6-1. Exceptions and Interrupts at Page 6-13 Vol. 1
// of Intel 64 and IA-32 Architectures Software Developer's Manual
/// Exception id for #DB
pub(crate) const DB_EX_ID: u32 = 1;
/// Exception id for #BP - triggered by the INT3 instruction
pub(crate) const BP_EX_ID: u32 = 3;

/// Software Breakpoint size in memory
pub(crate) const SW_BP_SIZE: usize = 1;
/// Software Breakpoint opcode - INT3
/// Check page 7-28 Vol. 3A of Intel 64 and IA-32
/// Architectures Software Developer's Manual
pub(crate) const SW_BP_OP: u8 = 0xCC;
/// Software Breakpoint written to memory
pub(crate) const SW_BP: [u8; SW_BP_SIZE] = [SW_BP_OP];
/// Maximum number of supported hardware breakpoints
pub(crate) const MAX_NO_OF_HW_BP: usize = 4;

/// Check page 19-4 Vol. 3B of Intel 64 and IA-32
/// Architectures Software Developer's Manual
/// Bit position of BS flag in DR6 debug register
pub(crate) const DR6_BS_FLAG_POS: usize = 14;
/// Bit mask of BS flag in DR6 debug register
pub(crate) const DR6_BS_FLAG_MASK: u64 = 1 << DR6_BS_FLAG_POS;
/// Bit position of HW breakpoints status in DR6 debug register
pub(crate) const DR6_HW_BP_FLAGS_POS: usize = 0;
/// Bit mask of HW breakpoints status in DR6 debug register
pub(crate) const DR6_HW_BP_FLAGS_MASK: u64 = 0x0F << DR6_HW_BP_FLAGS_POS;

/// Determine the reason the vCPU stopped
/// This is done by checking the DR6 register and the exception id
pub(crate) fn vcpu_stop_reason(
vm: &dyn DebuggableVm,
dr6: u64,
exception: u32,
) -> std::result::Result<VcpuStopReason, VcpuStopReasonError> {
let CommonRegisters { rip, .. } = vm.regs()?;
if DB_EX_ID == exception {
// If the BS flag in DR6 register is set, it means a single step
// instruction triggered the exit
// Check page 19-4 Vol. 3B of Intel 64 and IA-32
// Architectures Software Developer's Manual
if dr6 & DR6_BS_FLAG_MASK != 0 {
return Ok(VcpuStopReason::DoneStep);
}

// If any of the B0-B3 flags in DR6 register is set, it means a
// hardware breakpoint triggered the exit
// Check page 19-4 Vol. 3B of Intel 64 and IA-32
// Architectures Software Developer's Manual
if DR6_HW_BP_FLAGS_MASK & dr6 != 0 {
return Ok(VcpuStopReason::HwBp);
}
}

if BP_EX_ID == exception {
return Ok(VcpuStopReason::SwBp);
}

// Log an error and provide internal debugging info
tracing::error!(
r"The vCPU exited because of an unknown reason:
rip: {:?}
dr6: {:?}
exception: {:?}
",
rip,
dr6,
exception,
);

Ok(VcpuStopReason::Unknown)
}
#[cfg(target_arch = "aarch64")]
mod aarch64;
#[cfg(target_arch = "x86_64")]
mod x86_64;

#[cfg(target_arch = "aarch64")]
pub(crate) use aarch64::*;
#[cfg(target_arch = "x86_64")]
pub(crate) use x86_64::*;
Loading
Loading