From 94cdd06e726ddc708e49968006742cc28e77ac3c Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 20 Jul 2026 18:12:40 +0200 Subject: [PATCH 01/13] style(memtrack): disallow short functions on one line and reformat Set AllowShortFunctionsOnASingleLine: None in .clang-format and apply it, reformatting the allocator uprobe macros accordingly. --- crates/memtrack/.clang-format | 1 + crates/memtrack/src/ebpf/c/allocator.h | 34 ++++++++++++++------------ 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/crates/memtrack/.clang-format b/crates/memtrack/.clang-format index b687ea40..4755a340 100644 --- a/crates/memtrack/.clang-format +++ b/crates/memtrack/.clang-format @@ -4,3 +4,4 @@ BasedOnStyle: Google PointerAlignment: Left ColumnLimit: 100 IndentWidth: 4 +AllowShortFunctionsOnASingleLine: None diff --git a/crates/memtrack/src/ebpf/c/allocator.h b/crates/memtrack/src/ebpf/c/allocator.h index 440d7a87..88566b36 100644 --- a/crates/memtrack/src/ebpf/c/allocator.h +++ b/crates/memtrack/src/ebpf/c/allocator.h @@ -5,22 +5,24 @@ #include "utils/map_helpers.h" #include "utils/process_tracking.h" -#define UPROBE_ARG_RET(name, arg_expr, submit_block) \ - BPF_HASH_MAP(name##_arg, __u64, __u64, 10000); \ - SEC("uprobe") \ - int uprobe_##name(struct pt_regs* ctx) { return store_param(&name##_arg, arg_expr); } \ - SEC("uretprobe") \ - int uretprobe_##name(struct pt_regs* ctx) { \ - __u64* arg_ptr = take_param(&name##_arg); \ - if (!arg_ptr) { \ - return 0; \ - } \ - __u64 ret_val = PT_REGS_RC(ctx); \ - if (ret_val == 0) { \ - return 0; \ - } \ - __u64 arg0 = *arg_ptr; \ - submit_block; \ +#define UPROBE_ARG_RET(name, arg_expr, submit_block) \ + BPF_HASH_MAP(name##_arg, __u64, __u64, 10000); \ + SEC("uprobe") \ + int uprobe_##name(struct pt_regs* ctx) { \ + return store_param(&name##_arg, arg_expr); \ + } \ + SEC("uretprobe") \ + int uretprobe_##name(struct pt_regs* ctx) { \ + __u64* arg_ptr = take_param(&name##_arg); \ + if (!arg_ptr) { \ + return 0; \ + } \ + __u64 ret_val = PT_REGS_RC(ctx); \ + if (ret_val == 0) { \ + return 0; \ + } \ + __u64 arg0 = *arg_ptr; \ + submit_block; \ } #define UPROBE_RET(name, arg_expr, submit_block) \ From 8d321afd7d5ba91f2b192bfc962b7976112ccba6 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 13 Jul 2026 14:44:44 +0200 Subject: [PATCH 02/13] feat(memtrack): track RSS via kmem:rss_stat tracepoint Sample the kernel's per-mm resident counter through the kmem:rss_stat tracepoint, emitting absolute byte values per mm member. Adds the EVENT_TYPE_RSS contract, MemtrackEventKind::Rss, the parser arm, and a writer bench case. An rss_stat update from reclaim or another process's madvise fires in the actor's context; track (mm_id, member) -> owning pid so those updates reach the owner. External events may only lower a counter, so stale reads and mm_id collisions cannot invent peaks. --- crates/memtrack/src/ebpf/c/event.h | 6 ++ crates/memtrack/src/ebpf/c/main.bpf.c | 1 + crates/memtrack/src/ebpf/c/rss.bpf.h | 74 +++++++++++++++++++ .../memtrack/src/ebpf/c/utils/event_helpers.h | 19 ++++- crates/memtrack/src/ebpf/events.rs | 37 ++++++++++ crates/memtrack/src/ebpf/memtrack/tracking.rs | 4 + .../runner-shared/benches/memtrack_writer.rs | 6 +- .../src/artifacts/memtrack/mod.rs | 14 ++++ 8 files changed, 156 insertions(+), 5 deletions(-) create mode 100644 crates/memtrack/src/ebpf/c/rss.bpf.h diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index ff41f0be..89974526 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -9,6 +9,7 @@ #define EVENT_TYPE_MMAP 6 #define EVENT_TYPE_MUNMAP 7 #define EVENT_TYPE_BRK 8 +#define EVENT_TYPE_RSS 9 /* Common header shared by all event types */ struct event_header { @@ -45,6 +46,11 @@ struct event { uint64_t addr; /* address of mapping */ uint64_t size; /* size of mapping */ } mmap; + + struct { + int32_t member; + uint64_t size; + } rss; } data; }; diff --git a/crates/memtrack/src/ebpf/c/main.bpf.c b/crates/memtrack/src/ebpf/c/main.bpf.c index 1c5ad8d1..e46a144b 100644 --- a/crates/memtrack/src/ebpf/c/main.bpf.c +++ b/crates/memtrack/src/ebpf/c/main.bpf.c @@ -8,6 +8,7 @@ #include "allocator.h" #include "attach.h" #include "event.h" +#include "rss.bpf.h" #include "utils/event_helpers.h" #include "utils/map_helpers.h" #include "utils/process_tracking.h" diff --git a/crates/memtrack/src/ebpf/c/rss.bpf.h b/crates/memtrack/src/ebpf/c/rss.bpf.h new file mode 100644 index 00000000..822af655 --- /dev/null +++ b/crates/memtrack/src/ebpf/c/rss.bpf.h @@ -0,0 +1,74 @@ +#ifndef __RSS_BPF_H__ +#define __RSS_BPF_H__ + +#include "event.h" +#include "utils/event_helpers.h" +#include "utils/process_tracking.h" + +/* (rss_stat mm_id << 32 | member) -> {owning tgid, last in-context size}. Keyed per + * counter so an external (curr==0) update is attributed only once that mm/member was + * established in-context. An external event may only lower a counter: any size above + * the last in-context value is dropped, so neither a stale/racing reclaim read nor an + * mm_id hash collision with another task can invent a peak. LRU eviction + re-seeding + * on the owner's next in-context event covers hash reuse, so no teardown hook needed. */ +struct rss_owner { + __u32 pid; + __u64 size; +}; +struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __uint(max_entries, 40960); + __type(key, __u64); + __type(value, struct rss_owner); +} mm_to_pid SEC(".maps"); + +static __always_inline int submit_rss_event(__u32 owner_pid, __s32 member, __u64 size) { + SUBMIT_EVENT_AS(owner_pid, EVENT_TYPE_RSS, { + e->data.rss.member = member; + e->data.rss.size = size; + }); +} + +SEC("tracepoint/kmem/rss_stat") +int tracepoint_rss_stat(struct trace_event_raw_rss_stat* ctx) { + if (ctx->member == MM_SWAPENTS) { + return 0; + } + + __u32 cur = bpf_get_current_pid_tgid() >> 32; + __u64 key = ((__u64)ctx->mm_id << 32) | (__u32)ctx->member; + __u64 size = ctx->size; + __u32 owner; + + if (ctx->curr) { + if (!is_tracked(cur)) { + return 0; + } + owner = cur; + struct rss_owner state = {.pid = cur, .size = size}; + bpf_map_update_elem(&mm_to_pid, &key, &state, BPF_ANY); + } else { + struct rss_owner* found = bpf_map_lookup_elem(&mm_to_pid, &key); + if (!found) { + return 0; + } + owner = found->pid; + /* The owner's own teardown also presents as curr==0 (current->mm is cleared + * on exit), so drop it. Genuine external actors (reclaim, another process's + * madvise) run in a different task, so cur != owner. */ + if (cur == owner) { + return 0; + } + /* An external actor may only lower a counter. A larger value is a stale + * reclaim read or an mm_id hash collision with another task; dropping it + * keeps the reconstructed peak identical to the in-context timeline. */ + if (size > found->size) { + return 0; + } + found->size = size; + } + + return submit_rss_event(owner, ctx->member, size); +} + +#endif /* __RSS_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/c/utils/event_helpers.h b/crates/memtrack/src/ebpf/c/utils/event_helpers.h index b5d79b37..1be39e7e 100644 --- a/crates/memtrack/src/ebpf/c/utils/event_helpers.h +++ b/crates/memtrack/src/ebpf/c/utils/event_helpers.h @@ -38,12 +38,11 @@ static __always_inline __u64* take_param(void* map) { return value; } -#define SUBMIT_EVENT(evt_type, fill_data) \ +#define SUBMIT_EVENT_AS(owner_pid, evt_type, fill_data) \ { \ __u64 tid = bpf_get_current_pid_tgid(); \ - __u32 pid = tid >> 32; \ \ - if (!is_tracked(pid) || !is_enabled()) { \ + if (!is_enabled()) { \ return 0; \ } \ \ @@ -58,7 +57,7 @@ static __always_inline __u64* take_param(void* map) { } \ \ e->header.timestamp = bpf_ktime_get_ns(); \ - e->header.pid = pid; \ + e->header.pid = owner_pid; \ e->header.tid = tid & 0xFFFFFFFF; \ e->header.event_type = evt_type; \ \ @@ -68,6 +67,18 @@ static __always_inline __u64* take_param(void* map) { return 0; \ } +#define SUBMIT_EVENT(evt_type, fill_data) \ + { \ + __u64 tid = bpf_get_current_pid_tgid(); \ + __u32 pid = tid >> 32; \ + \ + if (!is_tracked(pid)) { \ + return 0; \ + } \ + \ + SUBMIT_EVENT_AS(pid, evt_type, fill_data); \ + } + static __always_inline int submit_alloc_event(__u64 size, __u64 addr) { SUBMIT_EVENT(EVENT_TYPE_MALLOC, { e->data.alloc.addr = addr; diff --git a/crates/memtrack/src/ebpf/events.rs b/crates/memtrack/src/ebpf/events.rs index f18216c6..cfe56ec8 100644 --- a/crates/memtrack/src/ebpf/events.rs +++ b/crates/memtrack/src/ebpf/events.rs @@ -74,6 +74,13 @@ pub fn parse_event(data: &[u8]) -> Option { size: event.data.mmap.size, }, ), + EVENT_TYPE_RSS => ( + 0, + MemtrackEventKind::Rss { + member: event.data.rss.member, + size: event.data.rss.size, + }, + ), unknown => { panic!("Unknown event type: {unknown}"); } @@ -185,4 +192,34 @@ mod tests { _ => panic!("Expected Malloc event kind"), } } + + #[test] + fn test_parse_rss_event() { + let mut event: bindings::event = unsafe { std::mem::zeroed() }; + event.header.event_type = bindings::EVENT_TYPE_RSS as u8; + event.header.timestamp = 12345678; + event.header.pid = 1000; + event.header.tid = 2000; + event.data.rss.member = 1; + event.data.rss.size = 4096 * 10; + + let bytes = unsafe { + std::slice::from_raw_parts( + &event as *const _ as *const u8, + std::mem::size_of_val(&event), + ) + }; + + let parsed = parse_event(bytes).unwrap(); + assert_eq!(parsed.pid, 1000); + assert_eq!(parsed.addr, 0); + + match parsed.kind { + MemtrackEventKind::Rss { member, size } => { + assert_eq!(member, 1); + assert_eq!(size, 4096 * 10); + } + _ => panic!("Expected Rss event kind"), + } + } } diff --git a/crates/memtrack/src/ebpf/memtrack/tracking.rs b/crates/memtrack/src/ebpf/memtrack/tracking.rs index dd2a5604..966d4eaa 100644 --- a/crates/memtrack/src/ebpf/memtrack/tracking.rs +++ b/crates/memtrack/src/ebpf/memtrack/tracking.rs @@ -4,9 +4,13 @@ use paste::paste; impl MemtrackBpf { attach_tracepoint!(sched_fork); + attach_tracepoint!(rss_stat); pub fn attach_tracepoints(&mut self) -> Result<()> { self.attach_sched_fork()?; + if let Err(e) = self.attach_rss_stat() { + warn!("Failed to attach rss_stat tracepoint, RSS collection disabled: {e:#}"); + } Ok(()) } diff --git a/crates/runner-shared/benches/memtrack_writer.rs b/crates/runner-shared/benches/memtrack_writer.rs index 62aa2451..17b4f172 100644 --- a/crates/runner-shared/benches/memtrack_writer.rs +++ b/crates/runner-shared/benches/memtrack_writer.rs @@ -13,7 +13,7 @@ fn generate_events(n: usize) -> Vec { let mut events = Vec::with_capacity(n); for _ in 0..n { let size = rng.gen_range(8..8192); - let kind = match rng.gen_range(0..8) { + let kind = match rng.gen_range(0..9) { 0 => MemtrackEventKind::Malloc { size }, 1 => MemtrackEventKind::Free, 2 => MemtrackEventKind::Realloc { @@ -25,6 +25,10 @@ fn generate_events(n: usize) -> Vec { 5 => MemtrackEventKind::Mmap { size }, 6 => MemtrackEventKind::Munmap { size }, 7 => MemtrackEventKind::Brk { size }, + 8 => MemtrackEventKind::Rss { + member: rng.gen_range(0..4), + size, + }, _ => unreachable!(), }; diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index ad843b28..d9a00923 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -78,6 +78,10 @@ pub enum MemtrackEventKind { Brk { size: u64, }, + Rss { + member: i32, + size: u64, + }, } pub struct MemtrackEventStream { @@ -116,6 +120,16 @@ mod tests { addr: 0x20, kind: MemtrackEventKind::Free, }, + MemtrackEvent { + pid: 1, + tid: 11, + timestamp: 300, + addr: 0, + kind: MemtrackEventKind::Rss { + member: 1, + size: 40960, + }, + }, ]; let artifact = MemtrackArtifact { From 55adca1ded2e48da6c96ac16e902c8d6e43eb4f6 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 17 Jul 2026 15:38:40 +0200 Subject: [PATCH 03/13] feat(memtrack): reconstruct RSS from gated folio rmap fentry hooks Attach fentry hooks on the folio-rmap add/remove functions, emitting signed page-count deltas per MM_* bucket so anon, file, and shmem RSS can be reconstructed over time. Gated behind CODSPEED_MEMTRACK_TRACK_RMAP; the programs stay autoload-off by default so the skeleton loads on any kernel. Adds the EVENT_TYPE_RMAP contract, MemtrackEventKind::Rmap, parser arm, and bench case. --- crates/memtrack/src/ebpf/c/event.h | 7 + crates/memtrack/src/ebpf/c/rss.bpf.h | 150 ++++++++++++++++++ crates/memtrack/src/ebpf/events.rs | 38 +++++ crates/memtrack/src/ebpf/memtrack/macros.rs | 15 ++ crates/memtrack/src/ebpf/memtrack/mod.rs | 112 ++++++++++++- crates/memtrack/src/ebpf/memtrack/tracking.rs | 69 ++++++++ crates/memtrack/src/ebpf/tracker.rs | 45 ++++-- .../runner-shared/benches/memtrack_writer.rs | 6 +- .../src/artifacts/memtrack/mod.rs | 4 + 9 files changed, 427 insertions(+), 19 deletions(-) diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index 89974526..6739c525 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -10,6 +10,7 @@ #define EVENT_TYPE_MUNMAP 7 #define EVENT_TYPE_BRK 8 #define EVENT_TYPE_RSS 9 +#define EVENT_TYPE_RMAP 10 /* Common header shared by all event types */ struct event_header { @@ -51,6 +52,12 @@ struct event { int32_t member; uint64_t size; } rss; + + struct { + int32_t member; /* MM_* counter index, same values as rss.member */ + int64_t delta; + uint64_t addr; + } rmap; } data; }; diff --git a/crates/memtrack/src/ebpf/c/rss.bpf.h b/crates/memtrack/src/ebpf/c/rss.bpf.h index 822af655..34b12d60 100644 --- a/crates/memtrack/src/ebpf/c/rss.bpf.h +++ b/crates/memtrack/src/ebpf/c/rss.bpf.h @@ -22,6 +22,10 @@ struct { __type(value, struct rss_owner); } mm_to_pid SEC(".maps"); +#define FOLIO_MAPPING_ANON 0x1UL + +const volatile __u32 page_shift = 12; + static __always_inline int submit_rss_event(__u32 owner_pid, __s32 member, __u64 size) { SUBMIT_EVENT_AS(owner_pid, EVENT_TYPE_RSS, { e->data.rss.member = member; @@ -71,4 +75,150 @@ int tracepoint_rss_stat(struct trace_event_raw_rss_stat* ctx) { return submit_rss_event(owner, ctx->member, size); } +/* Kernels < 6.18 store folio->flags as a bare unsigned long instead of + * memdesc_flags_t; probe which layout the running kernel has. */ +struct folio___legacy { + unsigned long flags; +} __attribute__((preserve_access_index)); + +static __always_inline unsigned long folio_read_flags(struct folio* folio) { + if (bpf_core_field_exists(folio->flags.f)) { + return BPF_CORE_READ(folio, flags).f; + } + return BPF_CORE_READ((struct folio___legacy*)folio, flags); +} + +/* Kernels < 6.6 store the large-folio order in a dedicated byte instead of + * the low byte of _flags_1. */ +struct folio___order_byte { + unsigned char _folio_order; +} __attribute__((preserve_access_index)); + +static __always_inline unsigned long folio_order(struct folio* folio) { + if (bpf_core_field_exists(((struct folio___order_byte*)folio)->_folio_order)) { + return BPF_CORE_READ((struct folio___order_byte*)folio, _folio_order); + } + return BPF_CORE_READ(folio, _flags_1) & 0xff; +} + +static __always_inline __u64 folio_nr_pages_est(struct folio* folio) { + unsigned long flags = folio_read_flags(folio); + if (!(flags & (1UL << bpf_core_enum_value(enum pageflags, PG_head)))) { + return 1; + } + unsigned long order = folio_order(folio); + return 1UL << order; +} + +static __always_inline int folio_is_anon(struct folio* folio) { + unsigned long mapping = (unsigned long)BPF_CORE_READ(folio, mapping); + return (mapping & FOLIO_MAPPING_ANON) != 0; +} + +/* Mirrors the kernel's mm_counter(): anon folios are also swapbacked, so the + * anon check must come first. */ +static __always_inline __s32 folio_mm_counter(struct folio* folio) { + if (folio_is_anon(folio)) { + return MM_ANONPAGES; + } + unsigned long flags = folio_read_flags(folio); + if (flags & (1UL << bpf_core_enum_value(enum pageflags, PG_swapbacked))) { + return MM_SHMEMPAGES; + } + return MM_FILEPAGES; +} + +static __always_inline __u64 folio_page_address(struct folio* folio, struct page* page, + struct vm_area_struct* vma) { + __u64 page_idx = ((__u64)page - (__u64)folio) / bpf_core_type_size(struct page); + __u64 pgoff = BPF_CORE_READ(folio, index) + page_idx; + __u64 vm_pgoff = BPF_CORE_READ(vma, vm_pgoff); + __u64 vm_start = BPF_CORE_READ(vma, vm_start); + if (pgoff < vm_pgoff) { + return vm_start; + } + return vm_start + ((pgoff - vm_pgoff) << page_shift); +} + +static __always_inline int submit_rmap(struct vm_area_struct* vma, __s32 member, __s64 delta, + __u64 addr) { + __u64 mm = (__u64)BPF_CORE_READ(vma, vm_mm); + struct task_struct* task = bpf_get_current_task_btf(); + if ((__u64)BPF_CORE_READ(task, mm) != mm) { + return 0; + } + + __u64 tid = bpf_get_current_pid_tgid(); + __u32 pid = tid >> 32; + if (!is_tracked(pid)) { + return 0; + } + + SUBMIT_EVENT_AS(pid, EVENT_TYPE_RMAP, { + e->data.rmap.member = member; + e->data.rmap.delta = delta; + e->data.rmap.addr = addr; + }); +} + +SEC("fentry/folio_add_new_anon_rmap") +int BPF_PROG(fentry_folio_add_new_anon_rmap, struct folio* folio, struct vm_area_struct* vma, + unsigned long address) { + return submit_rmap(vma, MM_ANONPAGES, (__s64)folio_nr_pages_est(folio), address); +} + +SEC("fentry/folio_add_anon_rmap_ptes") +int BPF_PROG(fentry_folio_add_anon_rmap_ptes, struct folio* folio, struct page* page, int nr_pages, + struct vm_area_struct* vma, unsigned long address) { + return submit_rmap(vma, MM_ANONPAGES, (__s64)nr_pages, address); +} + +SEC("fentry/folio_add_anon_rmap_pmd") +int BPF_PROG(fentry_folio_add_anon_rmap_pmd, struct folio* folio, struct page* page, + struct vm_area_struct* vma, unsigned long address) { + return submit_rmap(vma, MM_ANONPAGES, (__s64)folio_nr_pages_est(folio), address); +} + +SEC("fentry/folio_add_file_rmap_ptes") +int BPF_PROG(fentry_folio_add_file_rmap_ptes, struct folio* folio, struct page* page, int nr_pages, + struct vm_area_struct* vma) { + return submit_rmap(vma, folio_mm_counter(folio), (__s64)nr_pages, + folio_page_address(folio, page, vma)); +} + +SEC("fentry/folio_add_file_rmap_pmd") +int BPF_PROG(fentry_folio_add_file_rmap_pmd, struct folio* folio, struct page* page, + struct vm_area_struct* vma) { + return submit_rmap(vma, folio_mm_counter(folio), (__s64)folio_nr_pages_est(folio), + folio_page_address(folio, page, vma)); +} + +SEC("fentry/folio_add_file_rmap_pud") +int BPF_PROG(fentry_folio_add_file_rmap_pud, struct folio* folio, struct page* page, + struct vm_area_struct* vma) { + return submit_rmap(vma, folio_mm_counter(folio), (__s64)folio_nr_pages_est(folio), + folio_page_address(folio, page, vma)); +} + +SEC("fentry/folio_remove_rmap_ptes") +int BPF_PROG(fentry_folio_remove_rmap_ptes, struct folio* folio, struct page* page, int nr_pages, + struct vm_area_struct* vma) { + return submit_rmap(vma, folio_mm_counter(folio), -(__s64)nr_pages, + folio_page_address(folio, page, vma)); +} + +SEC("fentry/folio_remove_rmap_pmd") +int BPF_PROG(fentry_folio_remove_rmap_pmd, struct folio* folio, struct page* page, + struct vm_area_struct* vma) { + return submit_rmap(vma, folio_mm_counter(folio), -(__s64)folio_nr_pages_est(folio), + folio_page_address(folio, page, vma)); +} + +SEC("fentry/folio_remove_rmap_pud") +int BPF_PROG(fentry_folio_remove_rmap_pud, struct folio* folio, struct page* page, + struct vm_area_struct* vma) { + return submit_rmap(vma, folio_mm_counter(folio), -(__s64)folio_nr_pages_est(folio), + folio_page_address(folio, page, vma)); +} + #endif /* __RSS_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/events.rs b/crates/memtrack/src/ebpf/events.rs index cfe56ec8..8fe3a63d 100644 --- a/crates/memtrack/src/ebpf/events.rs +++ b/crates/memtrack/src/ebpf/events.rs @@ -81,6 +81,13 @@ pub fn parse_event(data: &[u8]) -> Option { size: event.data.rss.size, }, ), + EVENT_TYPE_RMAP => ( + event.data.rmap.addr, + MemtrackEventKind::Rmap { + member: event.data.rmap.member, + delta: event.data.rmap.delta, + }, + ), unknown => { panic!("Unknown event type: {unknown}"); } @@ -222,4 +229,35 @@ mod tests { _ => panic!("Expected Rss event kind"), } } + + #[test] + fn test_parse_rmap_event() { + let mut event: bindings::event = unsafe { std::mem::zeroed() }; + event.header.event_type = bindings::EVENT_TYPE_RMAP as u8; + event.header.timestamp = 12345678; + event.header.pid = 1000; + event.header.tid = 2000; + event.data.rmap.member = 3; + event.data.rmap.delta = 8; + event.data.rmap.addr = 0x7f00; + + let bytes = unsafe { + std::slice::from_raw_parts( + &event as *const _ as *const u8, + std::mem::size_of_val(&event), + ) + }; + + let parsed = parse_event(bytes).unwrap(); + assert_eq!(parsed.pid, 1000); + assert_eq!(parsed.addr, 0x7f00); + + match parsed.kind { + MemtrackEventKind::Rmap { member, delta } => { + assert_eq!(member, 3); + assert_eq!(delta, 8); + } + _ => panic!("Expected Rmap event kind"), + } + } } diff --git a/crates/memtrack/src/ebpf/memtrack/macros.rs b/crates/memtrack/src/ebpf/memtrack/macros.rs index 88da64a6..f4eca485 100644 --- a/crates/memtrack/src/ebpf/memtrack/macros.rs +++ b/crates/memtrack/src/ebpf/memtrack/macros.rs @@ -134,3 +134,18 @@ macro_rules! attach_tracepoint { } }; } + +macro_rules! attach_fentry { + ($func:ident, $prog:ident) => { + fn $func(&mut self) -> Result<()> { + let link = self + .skel + .progs + .$prog + .attach() + .context(format!("Failed to attach {} fentry", stringify!($prog)))?; + self.probes.push(link); + Ok(()) + } + }; +} diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 2faba0ec..f40f4c1b 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -1,8 +1,9 @@ use crate::prelude::*; -use libbpf_rs::Link; use libbpf_rs::skel::OpenSkel; use libbpf_rs::skel::SkelBuilder; +use libbpf_rs::{AsRawLibbpf, Link}; use std::collections::HashMap; +use std::ffi::CString; use std::mem::MaybeUninit; use std::path::Path; @@ -62,6 +63,27 @@ fn symbol_file_offset<'a>( Some((address - section.address() + sh_offset) as usize) } +fn page_shift() -> Result { + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + ensure!(page_size > 0, "Failed to read system page size"); + Ok((page_size as u32).trailing_zeros()) +} + +fn kernel_func_exists(btf: &libbpf_rs::btf::Btf, name: &str) -> bool { + let Ok(name) = CString::new(name) else { + return false; + }; + + let id = unsafe { + libbpf_rs::libbpf_sys::btf__find_by_name_kind( + btf.as_libbpf_object().as_ptr(), + name.as_ptr(), + libbpf_rs::libbpf_sys::BTF_KIND_FUNC, + ) + }; + id >= 0 +} + /// Attach targets resolved from a library's symbol tables. pub struct ResolvedSymbols { offsets: HashMap, @@ -76,16 +98,100 @@ impl ResolvedSymbols { pub struct MemtrackBpf { skel: Box>, probes: Vec, + track_rmap: bool, + btf_disabled_rmap_targets: Vec<&'static str>, } impl MemtrackBpf { pub fn new() -> Result { + let track_rmap = std::env::var("CODSPEED_MEMTRACK_TRACK_RMAP").is_ok_and(|v| v == "1"); + Self::new_with_rmap(track_rmap) + } + + pub fn new_with_rmap(track_rmap: bool) -> Result { let builder = MainSkelBuilder::default(); let open_object = Box::leak(Box::new(MaybeUninit::uninit())); - let open_skel = builder + let mut open_skel = builder .open(open_object) .context("Failed to open syscalls BPF skeleton")?; + open_skel + .maps + .rodata_data + .as_deref_mut() + .context("rodata map missing")? + .page_shift = page_shift()?; + let mut btf_disabled_rmap_targets = Vec::new(); + + if !track_rmap { + open_skel + .progs + .fentry_folio_add_new_anon_rmap + .set_autoload(false); + open_skel + .progs + .fentry_folio_add_anon_rmap_ptes + .set_autoload(false); + open_skel + .progs + .fentry_folio_add_anon_rmap_pmd + .set_autoload(false); + open_skel + .progs + .fentry_folio_add_file_rmap_ptes + .set_autoload(false); + open_skel + .progs + .fentry_folio_add_file_rmap_pmd + .set_autoload(false); + open_skel + .progs + .fentry_folio_add_file_rmap_pud + .set_autoload(false); + open_skel + .progs + .fentry_folio_remove_rmap_ptes + .set_autoload(false); + open_skel + .progs + .fentry_folio_remove_rmap_pmd + .set_autoload(false); + open_skel + .progs + .fentry_folio_remove_rmap_pud + .set_autoload(false); + } else { + let btf = libbpf_rs::btf::Btf::from_vmlinux().context("Failed to load vmlinux BTF")?; + macro_rules! disable_missing_kernel_func { + ($prog:ident, $target:literal) => { + if !kernel_func_exists(&btf, $target) { + open_skel.progs.$prog.set_autoload(false); + btf_disabled_rmap_targets.push($target); + warn!( + "Kernel function {} not present in BTF, disabling rmap fentry", + $target + ); + } + }; + } + + disable_missing_kernel_func!(fentry_folio_add_new_anon_rmap, "folio_add_new_anon_rmap"); + disable_missing_kernel_func!( + fentry_folio_add_anon_rmap_ptes, + "folio_add_anon_rmap_ptes" + ); + disable_missing_kernel_func!(fentry_folio_add_anon_rmap_pmd, "folio_add_anon_rmap_pmd"); + disable_missing_kernel_func!( + fentry_folio_add_file_rmap_ptes, + "folio_add_file_rmap_ptes" + ); + disable_missing_kernel_func!(fentry_folio_add_file_rmap_pmd, "folio_add_file_rmap_pmd"); + disable_missing_kernel_func!(fentry_folio_add_file_rmap_pud, "folio_add_file_rmap_pud"); + disable_missing_kernel_func!(fentry_folio_remove_rmap_ptes, "folio_remove_rmap_ptes"); + disable_missing_kernel_func!(fentry_folio_remove_rmap_pmd, "folio_remove_rmap_pmd"); + disable_missing_kernel_func!(fentry_folio_remove_rmap_pud, "folio_remove_rmap_pud"); + } + let skel = Box::new( open_skel .load() @@ -95,6 +201,8 @@ impl MemtrackBpf { Ok(Self { skel, probes: Vec::new(), + track_rmap, + btf_disabled_rmap_targets, }) } diff --git a/crates/memtrack/src/ebpf/memtrack/tracking.rs b/crates/memtrack/src/ebpf/memtrack/tracking.rs index 966d4eaa..70fc7077 100644 --- a/crates/memtrack/src/ebpf/memtrack/tracking.rs +++ b/crates/memtrack/src/ebpf/memtrack/tracking.rs @@ -5,12 +5,81 @@ use paste::paste; impl MemtrackBpf { attach_tracepoint!(sched_fork); attach_tracepoint!(rss_stat); + attach_fentry!( + attach_fentry_folio_add_new_anon_rmap, + fentry_folio_add_new_anon_rmap + ); + attach_fentry!( + attach_fentry_folio_add_anon_rmap_ptes, + fentry_folio_add_anon_rmap_ptes + ); + attach_fentry!( + attach_fentry_folio_add_anon_rmap_pmd, + fentry_folio_add_anon_rmap_pmd + ); + attach_fentry!( + attach_fentry_folio_add_file_rmap_ptes, + fentry_folio_add_file_rmap_ptes + ); + attach_fentry!( + attach_fentry_folio_add_file_rmap_pmd, + fentry_folio_add_file_rmap_pmd + ); + attach_fentry!( + attach_fentry_folio_add_file_rmap_pud, + fentry_folio_add_file_rmap_pud + ); + attach_fentry!( + attach_fentry_folio_remove_rmap_ptes, + fentry_folio_remove_rmap_ptes + ); + attach_fentry!( + attach_fentry_folio_remove_rmap_pmd, + fentry_folio_remove_rmap_pmd + ); + attach_fentry!( + attach_fentry_folio_remove_rmap_pud, + fentry_folio_remove_rmap_pud + ); + + fn rmap_target_enabled(&self, target: &str) -> bool { + !self.btf_disabled_rmap_targets.contains(&target) + } pub fn attach_tracepoints(&mut self) -> Result<()> { self.attach_sched_fork()?; if let Err(e) = self.attach_rss_stat() { warn!("Failed to attach rss_stat tracepoint, RSS collection disabled: {e:#}"); } + if self.track_rmap { + if self.rmap_target_enabled("folio_add_new_anon_rmap") { + self.attach_fentry_folio_add_new_anon_rmap()?; + } + if self.rmap_target_enabled("folio_add_anon_rmap_ptes") { + self.attach_fentry_folio_add_anon_rmap_ptes()?; + } + if self.rmap_target_enabled("folio_add_anon_rmap_pmd") { + self.attach_fentry_folio_add_anon_rmap_pmd()?; + } + if self.rmap_target_enabled("folio_add_file_rmap_ptes") { + self.attach_fentry_folio_add_file_rmap_ptes()?; + } + if self.rmap_target_enabled("folio_add_file_rmap_pmd") { + self.attach_fentry_folio_add_file_rmap_pmd()?; + } + if self.rmap_target_enabled("folio_add_file_rmap_pud") { + self.attach_fentry_folio_add_file_rmap_pud()?; + } + if self.rmap_target_enabled("folio_remove_rmap_ptes") { + self.attach_fentry_folio_remove_rmap_ptes()?; + } + if self.rmap_target_enabled("folio_remove_rmap_pmd") { + self.attach_fentry_folio_remove_rmap_pmd()?; + } + if self.rmap_target_enabled("folio_remove_rmap_pud") { + self.attach_fentry_folio_remove_rmap_pud()?; + } + } Ok(()) } diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index 11ecaf04..ff6c0881 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -33,10 +33,26 @@ impl Tracker { }) } + /// Track per-process RSS via the rss_stat tracepoint and folio rmap fentry + /// hooks without attaching any allocator probes. No exec-mapping watcher + /// runs, so `spawn` arms no on-demand allocator attachment — the tracker + /// observes only tracepoints and (when `track_rmap`) the rmap fentries. + pub fn new_without_allocators_with_rmap(track_rmap: bool) -> Result { + Self::bump_memlock_rlimit()?; + + let mut bpf = MemtrackBpf::new_with_rmap(track_rmap)?; + bpf.attach_tracepoints()?; + + Ok(Self { + bpf: Arc::new(Mutex::new(bpf)), + worker: Mutex::new(None), + }) + } + /// Spawn `cmd` under tracking: the target is wrapped so it stops itself - /// before exec'ing, its pid is armed while stopped, then it is resumed. - /// The watcher observes the target's own `execve` mappings — no allocation - /// escapes untracked. + /// before exec'ing, its pid is armed while stopped, then it is resumed. When + /// the tracker runs an exec-mapping watcher, arming the pid before resume + /// ensures no allocation mapping escapes untracked. /// /// `uid_gid` drops the child's privileges (a `Command`'s uid/gid cannot be /// read back, so it cannot be preserved through the wrap). @@ -48,11 +64,9 @@ impl Tracker { let child = spawn_stopped(&mut wrapped)?; let pid = child.id() as i32; - self.worker - .lock() - .as_ref() - .context("tracker already finished")? - .set_root_pid(pid); + if let Some(worker) = self.worker.lock().as_ref() { + worker.set_root_pid(pid); + } let (tx, rx) = mpsc::channel(); let poller = { @@ -81,15 +95,14 @@ impl Tracker { self.bpf.lock().dropped_events_count() } - /// Stop the attach worker and surface any fatal error it recorded, - /// including missed exec mappings (incomplete allocator coverage). + /// Stop the attach worker, if any, and surface any fatal error it recorded, + /// including missed exec mappings (incomplete allocator coverage). A tracker + /// without an allocator watcher has no worker, so this is a no-op. pub fn finish(&self) -> Result<()> { - let worker = self - .worker - .lock() - .take() - .context("tracker already finished")?; - worker.finish() + match self.worker.lock().take() { + Some(worker) => worker.finish(), + None => Ok(()), + } } /// Detach all attached probes. Called explicitly at teardown because the diff --git a/crates/runner-shared/benches/memtrack_writer.rs b/crates/runner-shared/benches/memtrack_writer.rs index 17b4f172..a6c610e8 100644 --- a/crates/runner-shared/benches/memtrack_writer.rs +++ b/crates/runner-shared/benches/memtrack_writer.rs @@ -13,7 +13,7 @@ fn generate_events(n: usize) -> Vec { let mut events = Vec::with_capacity(n); for _ in 0..n { let size = rng.gen_range(8..8192); - let kind = match rng.gen_range(0..9) { + let kind = match rng.gen_range(0..10) { 0 => MemtrackEventKind::Malloc { size }, 1 => MemtrackEventKind::Free, 2 => MemtrackEventKind::Realloc { @@ -29,6 +29,10 @@ fn generate_events(n: usize) -> Vec { member: rng.gen_range(0..4), size, }, + 9 => MemtrackEventKind::Rmap { + member: rng.gen_range(0..4), + delta: rng.gen_range(-1024..1024), + }, _ => unreachable!(), }; diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index d9a00923..aba8b8cc 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -82,6 +82,10 @@ pub enum MemtrackEventKind { member: i32, size: u64, }, + Rmap { + member: i32, + delta: i64, + }, } pub struct MemtrackEventStream { From eeb2f1f7c2bcb2e281e600e23c5541f350f2dcbc Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 14 Jul 2026 18:24:39 +0200 Subject: [PATCH 04/13] feat(memtrack): emit fork/exec/exit lifecycle events A forked child's inherited RSS is invisible to rss_stat: the fork-time counter copies fire outside the child's context, and anon COW faults are counter-neutral, so a child that only touches inherited memory never reports anything on its own. A fork event carrying the parent pid lets consumers seed the child from the parent's last absolutes; exec and exit mark where the address space is replaced or torn down. --- crates/memtrack/src/ebpf/c/event.h | 8 ++++ crates/memtrack/src/ebpf/c/rss.bpf.h | 43 +++++++++++++++++++ crates/memtrack/src/ebpf/events.rs | 35 +++++++++++++++ crates/memtrack/src/ebpf/memtrack/tracking.rs | 6 +++ .../src/artifacts/memtrack/mod.rs | 5 +++ 5 files changed, 97 insertions(+) diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index 6739c525..5dabffbe 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -11,6 +11,9 @@ #define EVENT_TYPE_BRK 8 #define EVENT_TYPE_RSS 9 #define EVENT_TYPE_RMAP 10 +#define EVENT_TYPE_FORK 11 +#define EVENT_TYPE_EXEC 12 +#define EVENT_TYPE_EXIT 13 /* Common header shared by all event types */ struct event_header { @@ -58,6 +61,11 @@ struct event { int64_t delta; uint64_t addr; } rmap; + + /* Process lifecycle events (fork carries the parent; exec/exit have no payload) */ + struct { + uint32_t parent_pid; + } fork; } data; }; diff --git a/crates/memtrack/src/ebpf/c/rss.bpf.h b/crates/memtrack/src/ebpf/c/rss.bpf.h index 34b12d60..ca208bf3 100644 --- a/crates/memtrack/src/ebpf/c/rss.bpf.h +++ b/crates/memtrack/src/ebpf/c/rss.bpf.h @@ -221,4 +221,47 @@ int BPF_PROG(fentry_folio_remove_rmap_pud, struct folio* folio, struct page* pag folio_page_address(folio, page, vma)); } +/* == Process lifecycle events == + * + * FORK lets userland seed a child's RSS from its parent at fork time: the + * kernel copies the mm counters during dup_mmap, but those updates fire + * rss_stat out of the child's context and anon COW faults are + * counter-neutral, so a child that only touches inherited memory never + * reports its RSS on its own. EXEC and EXIT mark the points where the + * address space is replaced or torn down, so userland resets to zero. + */ + +#define CLONE_THREAD 0x00010000 + +SEC("tracepoint/task/task_newtask") +int tracepoint_task_newtask(struct trace_event_raw_task_newtask* ctx) { + if (ctx->clone_flags & CLONE_THREAD) { + return 0; + } + + __u64 tid = bpf_get_current_pid_tgid(); + __u32 parent_pid = tid >> 32; + if (!is_tracked(parent_pid)) { + return 0; + } + + __u32 child_pid = ctx->pid; + SUBMIT_EVENT_AS(child_pid, EVENT_TYPE_FORK, { e->data.fork.parent_pid = parent_pid; }); +} + +SEC("tracepoint/sched/sched_process_exec") +int tracepoint_sched_process_exec(void* ctx) { + SUBMIT_EVENT(EVENT_TYPE_EXEC, {}); +} + +SEC("tracepoint/sched/sched_process_exit") +int tracepoint_sched_process_exit(void* ctx) { + __u64 tid = bpf_get_current_pid_tgid(); + if ((tid >> 32) != (tid & 0xFFFFFFFF)) { + return 0; + } + + SUBMIT_EVENT(EVENT_TYPE_EXIT, {}); +} + #endif /* __RSS_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/events.rs b/crates/memtrack/src/ebpf/events.rs index 8fe3a63d..ccab7f5f 100644 --- a/crates/memtrack/src/ebpf/events.rs +++ b/crates/memtrack/src/ebpf/events.rs @@ -88,6 +88,14 @@ pub fn parse_event(data: &[u8]) -> Option { delta: event.data.rmap.delta, }, ), + EVENT_TYPE_FORK => ( + 0, + MemtrackEventKind::Fork { + parent_pid: event.data.fork.parent_pid as i32, + }, + ), + EVENT_TYPE_EXEC => (0, MemtrackEventKind::Exec), + EVENT_TYPE_EXIT => (0, MemtrackEventKind::Exit), unknown => { panic!("Unknown event type: {unknown}"); } @@ -260,4 +268,31 @@ mod tests { _ => panic!("Expected Rmap event kind"), } } + + #[test] + fn test_parse_fork_event() { + let mut event: bindings::event = unsafe { std::mem::zeroed() }; + event.header.event_type = bindings::EVENT_TYPE_FORK as u8; + event.header.timestamp = 12345678; + event.header.pid = 1001; + event.header.tid = 2000; + event.data.fork.parent_pid = 1000; + + let bytes = unsafe { + std::slice::from_raw_parts( + &event as *const _ as *const u8, + std::mem::size_of_val(&event), + ) + }; + + let parsed = parse_event(bytes).unwrap(); + assert_eq!(parsed.pid, 1001); + + match parsed.kind { + MemtrackEventKind::Fork { parent_pid } => { + assert_eq!(parent_pid, 1000); + } + _ => panic!("Expected Fork event kind"), + } + } } diff --git a/crates/memtrack/src/ebpf/memtrack/tracking.rs b/crates/memtrack/src/ebpf/memtrack/tracking.rs index 70fc7077..bc8156c4 100644 --- a/crates/memtrack/src/ebpf/memtrack/tracking.rs +++ b/crates/memtrack/src/ebpf/memtrack/tracking.rs @@ -5,6 +5,9 @@ use paste::paste; impl MemtrackBpf { attach_tracepoint!(sched_fork); attach_tracepoint!(rss_stat); + attach_tracepoint!(task_newtask); + attach_tracepoint!(sched_process_exec); + attach_tracepoint!(sched_process_exit); attach_fentry!( attach_fentry_folio_add_new_anon_rmap, fentry_folio_add_new_anon_rmap @@ -48,6 +51,9 @@ impl MemtrackBpf { pub fn attach_tracepoints(&mut self) -> Result<()> { self.attach_sched_fork()?; + self.attach_task_newtask()?; + self.attach_sched_process_exec()?; + self.attach_sched_process_exit()?; if let Err(e) = self.attach_rss_stat() { warn!("Failed to attach rss_stat tracepoint, RSS collection disabled: {e:#}"); } diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index aba8b8cc..94c911c4 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -86,6 +86,11 @@ pub enum MemtrackEventKind { member: i32, delta: i64, }, + Fork { + parent_pid: pid_t, + }, + Exec, + Exit, } pub struct MemtrackEventStream { From e391d9f4634bea77577d6d3b84b5f8525ef9b2ac Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 17 Jul 2026 16:22:09 +0200 Subject: [PATCH 05/13] test(memtrack): validate RSS and rmap reconstruction against /proc Add /proc-backed integration cases, the rmap late-enable case, and the shared harness helpers used across the rss test suite. --- crates/memtrack/Cargo.toml | 2 +- crates/memtrack/testdata/rss/anon.c | 17 + crates/memtrack/testdata/rss/file.c | 30 + crates/memtrack/testdata/rss/fork.c | 39 ++ crates/memtrack/testdata/rss/fork_idle.c | 49 ++ crates/memtrack/testdata/rss/madvise.c | 64 +++ crates/memtrack/testdata/rss/madvise_extern.c | 47 ++ crates/memtrack/testdata/rss/mremap_move.c | 31 ++ crates/memtrack/testdata/rss/munmap_hole.c | 40 ++ .../memtrack/testdata/rss/rmap_late_enable.c | 59 ++ crates/memtrack/testdata/rss/rss_report.h | 50 ++ crates/memtrack/testdata/rss/shmem.c | 22 + crates/memtrack/testdata/rss/triangle.c | 24 + crates/memtrack/tests/rss_tests.rs | 512 ++++++++++++++++++ crates/memtrack/tests/shared.rs | 97 +++- .../tests/snapshots/rss_tests__rss_anon.snap | 30 + .../tests/snapshots/rss_tests__rss_file.snap | 30 + .../tests/snapshots/rss_tests__rss_fork.snap | 42 ++ .../snapshots/rss_tests__rss_fork_idle.snap | 45 ++ .../snapshots/rss_tests__rss_madvise.snap | 30 + .../snapshots/rss_tests__rss_mremap_move.snap | 30 + .../snapshots/rss_tests__rss_munmap_hole.snap | 30 + .../tests/snapshots/rss_tests__rss_shmem.snap | 30 + .../snapshots/rss_tests__rss_triangle.snap | 30 + 24 files changed, 1377 insertions(+), 3 deletions(-) create mode 100644 crates/memtrack/testdata/rss/anon.c create mode 100644 crates/memtrack/testdata/rss/file.c create mode 100644 crates/memtrack/testdata/rss/fork.c create mode 100644 crates/memtrack/testdata/rss/fork_idle.c create mode 100644 crates/memtrack/testdata/rss/madvise.c create mode 100644 crates/memtrack/testdata/rss/madvise_extern.c create mode 100644 crates/memtrack/testdata/rss/mremap_move.c create mode 100644 crates/memtrack/testdata/rss/munmap_hole.c create mode 100644 crates/memtrack/testdata/rss/rmap_late_enable.c create mode 100644 crates/memtrack/testdata/rss/rss_report.h create mode 100644 crates/memtrack/testdata/rss/shmem.c create mode 100644 crates/memtrack/testdata/rss/triangle.c create mode 100644 crates/memtrack/tests/rss_tests.rs create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_anon.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_file.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_fork.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_fork_idle.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_madvise.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_mremap_move.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_munmap_hole.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_shmem.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_triangle.snap diff --git a/crates/memtrack/Cargo.toml b/crates/memtrack/Cargo.toml index c4311d9e..b3bd218e 100644 --- a/crates/memtrack/Cargo.toml +++ b/crates/memtrack/Cargo.toml @@ -46,7 +46,7 @@ bindgen = "0.72" tempfile = { workspace = true } rstest = { workspace = true } test-log = { workspace = true } -insta = { workspace = true } +insta = { workspace = true, features = ["json", "redactions"] } test-with = { workspace = true } [package.metadata.dist] diff --git a/crates/memtrack/testdata/rss/anon.c b/crates/memtrack/testdata/rss/anon.c new file mode 100644 index 00000000..8aa0974e --- /dev/null +++ b/crates/memtrack/testdata/rss/anon.c @@ -0,0 +1,17 @@ +#include +#include +#include + +#include "rss_report.h" + +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + size_t len = 64UL * 1024 * 1024; + void* mem = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) return 1; + memset(mem, 0x42, len); + int ret = write_rss_report(argv[1]); + munmap(mem, len); + return ret; +} diff --git a/crates/memtrack/testdata/rss/file.c b/crates/memtrack/testdata/rss/file.c new file mode 100644 index 00000000..5d45c293 --- /dev/null +++ b/crates/memtrack/testdata/rss/file.c @@ -0,0 +1,30 @@ +#include +#include +#include +#include +#include + +#include "rss_report.h" + +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + size_t len = 64UL * 1024 * 1024; + char template[] = "/tmp/memtrack-rss-file-XXXXXX"; + int fd = mkstemp(template); + if (fd < 0) return 1; + unlink(template); + char chunk[65536]; + memset(chunk, 0x42, sizeof(chunk)); + for (size_t off = 0; off < len; off += sizeof(chunk)) { + if (write(fd, chunk, sizeof(chunk)) != (ssize_t)sizeof(chunk)) return 1; + } + void* mem = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0); + if (mem == MAP_FAILED) return 1; + volatile char sink = 0; + for (size_t i = 0; i < len; i += 4096) sink ^= ((volatile char*)mem)[i]; + int ret = write_rss_report(argv[1]); + munmap(mem, len); + close(fd); + return ret + (sink & 0); +} diff --git a/crates/memtrack/testdata/rss/fork.c b/crates/memtrack/testdata/rss/fork.c new file mode 100644 index 00000000..f6fcfac0 --- /dev/null +++ b/crates/memtrack/testdata/rss/fork.c @@ -0,0 +1,39 @@ +#include +#include +#include +#include +#include + +#include "rss_report.h" + +static int append_rss(const char* path, const char* label) { + long kb = rss_status_kb("RssAnon:"); + if (kb < 0) return 1; + FILE* report = fopen(path, "a"); + if (!report) return 1; + fprintf(report, "%s: %ld\n", label, kb); + fclose(report); + return 0; +} + +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + size_t len = 0x20 * 1024 * 1024; + void* parent = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (parent == MAP_FAILED) return 1; + memset(parent, 0x42, len); + pid_t pid = fork(); + if (pid < 0) return 1; + if (pid == 0) { + void* child = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (child == MAP_FAILED) _exit(1); + memset(child, 0x24, len); + _exit(append_rss(argv[1], "ChildRssAnonKb")); + } + int status; + if (waitpid(pid, &status, 0) < 0) return 1; + int ret = append_rss(argv[1], "ParentRssAnonKb"); + munmap(parent, len); + return ret || !WIFEXITED(status) || WEXITSTATUS(status) != 0; +} diff --git a/crates/memtrack/testdata/rss/fork_idle.c b/crates/memtrack/testdata/rss/fork_idle.c new file mode 100644 index 00000000..0c5e5643 --- /dev/null +++ b/crates/memtrack/testdata/rss/fork_idle.c @@ -0,0 +1,49 @@ +#include +#include +#include +#include +#include + +#include "rss_report.h" + +/* The child only touches memory inherited from the parent: COW faults of anon + * pages are counter-neutral, so the child never reports its own RSS. Its + * footprint is only observable through fork-event seeding. The child must not + * allocate (no stdio/malloc), so the parent samples /proc//status while + * the child blocks on a pipe. */ +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + size_t len = 0x20 * 1024 * 1024; + void* mem = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) return 1; + memset(mem, 0x42, len); + + int ready[2]; + int release[2]; + if (pipe(ready) || pipe(release)) return 1; + + pid_t pid = fork(); + if (pid < 0) return 1; + if (pid == 0) { + memset(mem, 0x24, len); + char b = 1; + if (write(ready[1], &b, 1) != 1) _exit(1); + if (read(release[0], &b, 1) != 1) _exit(1); + _exit(0); + } + + char b; + if (read(ready[0], &b, 1) != 1) return 1; + long child_anon = rss_status_kb_pid(pid, "RssAnon:"); + int ret = write_rss_report(argv[1]); + FILE* report = fopen(argv[1], "a"); + if (!report) return 1; + fprintf(report, "ChildRssAnon: %ld\n", child_anon); + fclose(report); + if (write(release[1], &b, 1) != 1) return 1; + + int status; + if (waitpid(pid, &status, 0) < 0) return 1; + return ret || child_anon < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0; +} diff --git a/crates/memtrack/testdata/rss/madvise.c b/crates/memtrack/testdata/rss/madvise.c new file mode 100644 index 00000000..e5f77972 --- /dev/null +++ b/crates/memtrack/testdata/rss/madvise.c @@ -0,0 +1,64 @@ +#include +#include +#include +#include + +#include "rss_report.h" + +/* AnonHugePages for the whole process, from smaps_rollup (not in /proc/status). */ +static long anon_huge_pages_kb(void) { + FILE* rollup = fopen("/proc/self/smaps_rollup", "r"); + if (!rollup) return 0; + char line[256]; + long kb = 0; + while (fgets(line, sizeof(line), rollup)) { + if (sscanf(line, "AnonHugePages: %ld", &kb) == 1) break; + } + fclose(rollup); + return kb; +} + +/* Two 32 MiB anon regions: one advised MADV_HUGEPAGE (2 MiB-aligned so the + * kernel *may* fault it as PMD folios), one MADV_NOHUGEPAGE (guaranteed pte + * path). MADV_HUGEPAGE is only advisory, so nothing here depends on THP + * actually materializing: the accounted totals are identical either way. + * + * Both regions are dropped with MADV_DONTNEED and faulted a second time. If + * the in-context removes were missed, the reconstructed running total reaches + * 128 MiB instead of 64 MiB, so the snapshot peak is the assertion. The report + * also carries the observed AnonHugePages (hex, so it stays out of the + * snapshot) letting the test require PMD-sized rmap deltas exactly when THP + * actually materialized. */ +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + const size_t len = 32UL * 1024 * 1024; + const size_t align = 2UL * 1024 * 1024; + + void* raw = mmap(NULL, len + align, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (raw == MAP_FAILED) return 1; + char* huge = (char*)(((uintptr_t)raw + align - 1) & ~(uintptr_t)(align - 1)); + if (madvise(huge, len, MADV_HUGEPAGE) != 0) return 1; + + char* base = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (base == MAP_FAILED) return 1; + if (madvise(base, len, MADV_NOHUGEPAGE) != 0) return 1; + + memset(huge, 0x42, len); + memset(base, 0x42, len); + + if (madvise(huge, len, MADV_DONTNEED) != 0) return 1; + if (madvise(base, len, MADV_DONTNEED) != 0) return 1; + + memset(huge, 0x43, len); + memset(base, 0x43, len); + + int ret = write_rss_report(argv[1]); + FILE* report = fopen(argv[1], "a"); + if (!report) return 1; + fprintf(report, "ThpKb: 0x%lx\n", anon_huge_pages_kb()); + fclose(report); + munmap(raw, len + align); + munmap(base, len); + return ret; +} diff --git a/crates/memtrack/testdata/rss/madvise_extern.c b/crates/memtrack/testdata/rss/madvise_extern.c new file mode 100644 index 00000000..4be4c6b2 --- /dev/null +++ b/crates/memtrack/testdata/rss/madvise_extern.c @@ -0,0 +1,47 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char** argv) { + (void)argc; (void)argv; + sleep(1); /* let the tracker attach + enable + add root pid */ + + size_t len = 64UL * 1024 * 1024; + + char path[] = "/tmp/memtrack_madv_XXXXXX"; + int fd = mkstemp(path); + if (fd < 0) return 1; + unlink(path); + if (ftruncate(fd, len) != 0) return 1; + + void* mem = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0); + if (mem == MAP_FAILED) return 1; + + /* Fault the file pages into A's RSS (MM_FILEPAGES), in-context -> seeds ownership. */ + volatile char sink = 0; + for (size_t i = 0; i < len; i += 4096) sink ^= ((volatile char*)mem)[i]; + (void)sink; + + pid_t pid = fork(); + if (pid < 0) return 1; + if (pid == 0) { + /* B: page out A's (parent's) region from B's context. */ + int pidfd = syscall(SYS_pidfd_open, getppid(), 0); + if (pidfd < 0) _exit(1); + struct iovec iov = {.iov_base = mem, .iov_len = len}; + syscall(SYS_process_madvise, pidfd, &iov, 1UL, MADV_PAGEOUT, 0UL); + _exit(0); + } + + int status; + if (waitpid(pid, &status, 0) < 0) return 1; + sleep(1); /* let the external decrement flush to the ring buffer */ + /* No munmap: an in-context decrement would mask the external-path signal. */ + return 0; +} diff --git a/crates/memtrack/testdata/rss/mremap_move.c b/crates/memtrack/testdata/rss/mremap_move.c new file mode 100644 index 00000000..3b86fc03 --- /dev/null +++ b/crates/memtrack/testdata/rss/mremap_move.c @@ -0,0 +1,31 @@ +#define _GNU_SOURCE +#include +#include +#include + +#include "rss_report.h" + +/* Fault 32 MiB, then force mremap to move it to a reserved destination. The + * move relocates page tables without rmap remove/add, so no events should + * fire and the second memset must not refault: a peak of 64 MiB instead of + * 32 MiB means the move was double-counted or the pages were dropped. */ +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + const size_t len = 32UL * 1024 * 1024; + + char* src = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (src == MAP_FAILED) return 1; + memset(src, 0x42, len); + + void* reserved = mmap(NULL, len, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserved == MAP_FAILED) return 1; + + char* dst = mremap(src, len, len, MREMAP_MAYMOVE | MREMAP_FIXED, reserved); + if (dst == MAP_FAILED) return 1; + memset(dst, 0x43, len); + + int ret = write_rss_report(argv[1]); + munmap(dst, len); + return ret; +} diff --git a/crates/memtrack/testdata/rss/munmap_hole.c b/crates/memtrack/testdata/rss/munmap_hole.c new file mode 100644 index 00000000..e769ff5b --- /dev/null +++ b/crates/memtrack/testdata/rss/munmap_hole.c @@ -0,0 +1,40 @@ +#include +#include +#include +#include + +#include "rss_report.h" + +/* Fault 64 MiB, punch a 16 MiB hole with munmap, then map and fault the hole + * again. If the partial-unmap removes were missed (or covered the wrong + * range), the reconstructed running total peaks at 80 MiB instead of 64 MiB. + * MADV_NOHUGEPAGE keeps every folio a single page: the region is only + * page-aligned, so a straddling PMD folio would blur the hole boundaries. */ +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + const size_t len = 64UL * 1024 * 1024; + const size_t hole_off = 24UL * 1024 * 1024; + const size_t hole_len = 16UL * 1024 * 1024; + + char* mem = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) return 1; + if (madvise(mem, len, MADV_NOHUGEPAGE) != 0) return 1; + memset(mem, 0x42, len); + + if (munmap(mem + hole_off, hole_len) != 0) return 1; + + void* refill = mmap(mem + hole_off, hole_len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (refill == MAP_FAILED) return 1; + if (madvise(refill, hole_len, MADV_NOHUGEPAGE) != 0) return 1; + memset(refill, 0x43, hole_len); + + int ret = write_rss_report(argv[1]); + FILE* report = fopen(argv[1], "a"); + if (!report) return 1; + fprintf(report, "Layout: 0x%lx 0x%lx 0x%lx 0x%lx\n", (unsigned long)mem, hole_off, hole_len, + len); + fclose(report); + return ret; +} diff --git a/crates/memtrack/testdata/rss/rmap_late_enable.c b/crates/memtrack/testdata/rss/rmap_late_enable.c new file mode 100644 index 00000000..0bdae37b --- /dev/null +++ b/crates/memtrack/testdata/rss/rmap_late_enable.c @@ -0,0 +1,59 @@ +#include +#include +#include +#include + +#include "rss_report.h" + +/* Two-phase fixture that faults anonymous memory before and after a handshake: + * + * 1. Fault a baseline region, then signal `ready` and block on `go`. + * 2. After `go` appears, fault a second region of the same (anon) member. + * + * The handshake lets the caller act between the two phases (e.g. start + * observing only phase 2). Both regions touch MM_ANONPAGES, so an absolute + * counter read after phase 2 covers baseline + growth, while a delta observed + * only from phase 2 covers growth alone. + * + * argv: [1]=report path, [2]=ready path, [3]=go path. */ +static void touch(const char* path) { + FILE* f = fopen(path, "w"); + if (f) { + fclose(f); + } +} + +int main(int argc, char** argv) { + if (argc != 4) { + return 1; + } + const char* report_path = argv[1]; + const char* ready_path = argv[2]; + const char* go_path = argv[3]; + + size_t region = 64UL * 1024 * 1024; + + void* baseline = mmap(NULL, region, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (baseline == MAP_FAILED) { + return 1; + } + memset(baseline, 0x42, region); + + touch(ready_path); + while (access(go_path, F_OK) != 0) { + usleep(1000); + } + + void* growth = mmap(NULL, region, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (growth == MAP_FAILED) { + return 1; + } + memset(growth, 0x42, region); + + int ret = write_rss_report(report_path); + munmap(baseline, region); + munmap(growth, region); + return ret; +} diff --git a/crates/memtrack/testdata/rss/rss_report.h b/crates/memtrack/testdata/rss/rss_report.h new file mode 100644 index 00000000..e144d145 --- /dev/null +++ b/crates/memtrack/testdata/rss/rss_report.h @@ -0,0 +1,50 @@ +#ifndef RSS_REPORT_H +#define RSS_REPORT_H + +#include +#include +#include + +static long rss_status_kb_pid(int pid, const char* key) { + char status_path[64]; + snprintf(status_path, sizeof(status_path), "/proc/%d/status", pid); + FILE* status = fopen(status_path, "r"); + if (!status) return -1; + char line[256]; + long kb = -1; + size_t key_len = strlen(key); + while (fgets(line, sizeof(line), status)) { + if (strncmp(line, key, key_len) == 0 && sscanf(line + key_len, " %ld", &kb) == 1) { + break; + } + } + fclose(status); + return kb; +} + +static long rss_status_kb(const char* key) { + return rss_status_kb_pid(getpid(), key); +} + +static int write_rss_report(const char* path) { + long anon = rss_status_kb("RssAnon:"); + long file = rss_status_kb("RssFile:"); + long shmem = rss_status_kb("RssShmem:"); + /* VmHWM instead of getrusage(): ru_maxrss includes signal->maxrss, which + * survives execve and so reports the peak of the pre-exec parent image. + * VmHWM belongs to the mm and starts fresh at exec. */ + long max_rss = rss_status_kb("VmHWM:"); + if (anon < 0 || file < 0 || shmem < 0 || max_rss < 0) { + return 1; + } + FILE* report = fopen(path, "w"); + if (!report) { + return 1; + } + fprintf(report, "RssAnon: %ld\nRssFile: %ld\nRssShmem: %ld\nMaxRssKb: %ld\n", anon, file, + shmem, max_rss); + fclose(report); + return 0; +} + +#endif diff --git a/crates/memtrack/testdata/rss/shmem.c b/crates/memtrack/testdata/rss/shmem.c new file mode 100644 index 00000000..63b53e01 --- /dev/null +++ b/crates/memtrack/testdata/rss/shmem.c @@ -0,0 +1,22 @@ +#define _GNU_SOURCE +#include +#include +#include + +#include "rss_report.h" + +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + size_t len = 64UL * 1024 * 1024; + int fd = memfd_create("memtrack-rss-shmem", 0); + if (fd < 0) return 1; + if (ftruncate(fd, len) != 0) return 1; + void* mem = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (mem == MAP_FAILED) return 1; + memset(mem, 0x42, len); + int ret = write_rss_report(argv[1]); + munmap(mem, len); + close(fd); + return ret; +} diff --git a/crates/memtrack/testdata/rss/triangle.c b/crates/memtrack/testdata/rss/triangle.c new file mode 100644 index 00000000..69bf7bf6 --- /dev/null +++ b/crates/memtrack/testdata/rss/triangle.c @@ -0,0 +1,24 @@ +#include +#include +#include + +#include "rss_report.h" + +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + const size_t chunk = 16UL * 1024 * 1024; + void* bufs[4]; + for (int i = 0; i < 4; i++) { + bufs[i] = mmap(NULL, chunk, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (bufs[i] == MAP_FAILED) return 1; + memset(bufs[i], 0x42, chunk); + usleep(50 * 1000); + } + int ret = write_rss_report(argv[1]); + for (int i = 0; i < 4; i++) { + munmap(bufs[i], chunk); + usleep(50 * 1000); + } + return ret; +} diff --git a/crates/memtrack/tests/rss_tests.rs b/crates/memtrack/tests/rss_tests.rs new file mode 100644 index 00000000..8875096b --- /dev/null +++ b/crates/memtrack/tests/rss_tests.rs @@ -0,0 +1,512 @@ +#[macro_use] +mod shared; + +use itertools::Itertools; +use rstest::rstest; +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; +use serde::Serialize; +use std::collections::BTreeMap; +use std::process::Command; +use tempfile::TempDir; + +const MIB: u64 = 1024 * 1024; + +fn mib_16(bytes: u64) -> u64 { + (bytes + 8 * MIB) / (16 * MIB) * 16 +} + +fn parse_report(report: &str) -> BTreeMap { + report + .lines() + .filter_map(|line| { + let mut parts = line.split_whitespace(); + let key = parts.next()?.to_string(); + let kb: u64 = parts.next()?.parse().ok()?; + Some((key, mib_16(kb * 1024))) + }) + .collect() +} + +#[derive(Debug, Serialize)] +struct PidRss { + pid: i32, + file_mib: u64, + anon_mib: u64, + shmem_mib: u64, + max_rss_mib: u64, +} + +#[derive(Serialize)] +struct RssSummary { + report: BTreeMap, + rss_stat: Vec, + rmap: Vec, +} + +#[derive(Default)] +struct PeakAccum { + latest: [i64; 4], + peaks: [i64; 4], + max_rss: i64, +} + +impl PeakAccum { + /// Absolute assignment (rss_stat: the kernel counter's current value). + fn set(&mut self, index: usize, bytes: i64) { + self.latest[index] = bytes; + self.update_peaks(); + } + + /// Delta accumulation (rmap: summed folio add/remove deltas from zero). + fn add(&mut self, index: usize, delta_bytes: i64) { + self.latest[index] += delta_bytes; + self.update_peaks(); + } + + /// Fork: child inherits the parent's current resident values. + fn seed(&mut self, latest: [i64; 4]) { + self.latest = latest; + self.update_peaks(); + } + + /// Exec/Exit: reset the running value only; recorded peaks are retained + /// (an absolute rss_stat re-syncs on the next event; rmap re-accumulates). + fn reset(&mut self) { + self.latest = [0; 4]; + } + + fn update_peaks(&mut self) { + for (peak, latest) in self.peaks.iter_mut().zip(self.latest) { + *peak = (*peak).max(latest); + } + self.max_rss = self + .max_rss + .max(self.latest[0] + self.latest[1] + self.latest[3]); + } +} + +/// Reduce the raw event stream to per-pid resident peaks in BYTES. +/// Returns first-activity pid order plus rss_stat and rmap accumulators. +fn per_pid_raw( + events: &[MemtrackEvent], +) -> (Vec, BTreeMap, BTreeMap) { + // Pid values can wrap, so numeric order is not stable; both views emit + // their rows in one shared first-activity order, keeping the relative + // order of processes consistent between `rss_stat` and `rmap` after + // the pids are redacted. + let mut order: Vec = Vec::new(); + let mut rss: BTreeMap = BTreeMap::new(); + let mut rmap: BTreeMap = BTreeMap::new(); + + fn seen(order: &mut Vec, pid: i32) { + if !order.contains(&pid) { + order.push(pid); + } + } + + for event in events.iter().sorted_by_key(|event| event.timestamp) { + match event.kind { + MemtrackEventKind::Rss { member, size } => { + let Ok(index @ 0..4) = usize::try_from(member) else { + continue; + }; + seen(&mut order, event.pid); + rss.entry(event.pid).or_default().set(index, size as i64); + } + MemtrackEventKind::Rmap { member, delta } => { + let Ok(index @ 0..4) = usize::try_from(member) else { + continue; + }; + seen(&mut order, event.pid); + rmap.entry(event.pid) + .or_default() + .add(index, delta * page_size() as i64); + } + MemtrackEventKind::Fork { parent_pid } => { + seen(&mut order, event.pid); + let seed = rss.get(&parent_pid).map(|p| p.latest).unwrap_or_default(); + rss.entry(event.pid).or_default().seed(seed); + let seed = rmap.get(&parent_pid).map(|p| p.latest).unwrap_or_default(); + rmap.entry(event.pid).or_default().seed(seed); + } + MemtrackEventKind::Exec | MemtrackEventKind::Exit => { + if let Some(acc) = rss.get_mut(&event.pid) { + acc.reset(); + } + if let Some(acc) = rmap.get_mut(&event.pid) { + acc.reset(); + } + } + _ => {} + } + } + (order, rss, rmap) +} + +fn per_pid_peaks(events: &[MemtrackEvent]) -> (Vec, Vec) { + let (order, rss, rmap) = per_pid_raw(events); + let project = |map: &BTreeMap| -> Vec { + order + .iter() + .filter_map(|pid| { + let acc = map.get(pid)?; + Some(PidRss { + pid: *pid, + file_mib: mib_16(acc.peaks[0].max(0) as u64), + anon_mib: mib_16(acc.peaks[1].max(0) as u64), + shmem_mib: mib_16(acc.peaks[3].max(0) as u64), + max_rss_mib: mib_16(acc.max_rss.max(0) as u64), + }) + }) + .collect() + }; + (project(&rss), project(&rmap)) +} + +/// Compile a fixture that writes a `/proc` RSS report to its argv[1], run it under +/// `track`, and return the raw report text alongside the collected events. +/// +/// The report read is best-effort: some fixtures write no report. +fn track_fixture( + source: &str, + name: &str, + track: impl FnOnce(Command) -> shared::TrackResult, +) -> Result<(Option, Vec), Box> { + let temp_dir = TempDir::new()?; + std::fs::write( + temp_dir.path().join("rss_report.h"), + include_str!("../testdata/rss/rss_report.h"), + )?; + let binary = shared::compile_c_source(source, name, temp_dir.path())?; + let report_path = temp_dir.path().join(format!("{name}.report")); + let mut command = Command::new(&binary); + command.arg(&report_path); + + let (events, thread_handle) = track(command)?; + let raw_report = std::fs::read_to_string(&report_path).ok(); + thread_handle.join().unwrap(); + Ok((raw_report, events)) +} + +/// The first fork observed: `(parent_pid, child_pid)`. +fn first_fork_pair(events: &[MemtrackEvent]) -> Option<(i32, i32)> { + events.iter().find_map(|e| match e.kind { + MemtrackEventKind::Fork { parent_pid } => Some((parent_pid, e.pid)), + _ => None, + }) +} + +/// Pins reconstructed rmap addresses to the exact punched range: hole pages are +/// the only ones removed and later re-added; every other page in the region is +/// added first and only removed afterwards. +fn assert_rmap_hole_addresses( + events: &[MemtrackEvent], + base: u64, + hole_off: u64, + hole_len: u64, + len: u64, +) { + const PAGE: u64 = 4096; + let n_pages = (len / PAGE) as usize; + let mut first_remove = vec![u64::MAX; n_pages]; + let mut added = vec![false; n_pages]; + let mut readded = vec![false; n_pages]; + + for event in events.iter().sorted_by_key(|e| e.timestamp) { + let MemtrackEventKind::Rmap { delta, .. } = event.kind else { + continue; + }; + if event.addr < base || event.addr >= base + len { + continue; + } + let first = ((event.addr - base) / PAGE) as usize; + let last = (first + delta.unsigned_abs() as usize).min(n_pages); + for page in first..last { + if delta > 0 { + added[page] = true; + if event.timestamp > first_remove[page] { + readded[page] = true; + } + } else { + first_remove[page] = first_remove[page].min(event.timestamp); + } + } + } + + let hole = (hole_off / PAGE) as usize..((hole_off + hole_len) / PAGE) as usize; + for page in 0..n_pages { + assert!(added[page], "page {page} never saw an rmap add"); + assert_eq!( + readded[page], + hole.contains(&page), + "page {page}: remove-then-add pattern does not match the hole range" + ); + } +} + +#[test_with::env(GITHUB_ACTIONS)] +#[rstest] +#[case::anon(include_str!("../testdata/rss/anon.c"), "anon")] +#[case::file(include_str!("../testdata/rss/file.c"), "file")] +#[case::shmem(include_str!("../testdata/rss/shmem.c"), "shmem")] +#[case::fork(include_str!("../testdata/rss/fork.c"), "fork")] +#[case::fork_idle(include_str!("../testdata/rss/fork_idle.c"), "fork_idle")] +#[case::triangle(include_str!("../testdata/rss/triangle.c"), "triangle")] +#[case::madvise(include_str!("../testdata/rss/madvise.c"), "madvise")] +#[case::munmap_hole(include_str!("../testdata/rss/munmap_hole.c"), "munmap_hole")] +#[case::mremap_move(include_str!("../testdata/rss/mremap_move.c"), "mremap_move")] +fn test_rss_rmap_tracking( + #[case] source: &str, + #[case] name: &str, +) -> Result<(), Box> { + let (raw_report, events) = track_fixture(source, name, shared::track_command_with_rmap)?; + let raw_report = raw_report.ok_or("fixture wrote no rss report")?; + let (rss_stat, rmap) = per_pid_peaks(&events); + let summary = RssSummary { + report: parse_report(&raw_report), + rss_stat, + rmap, + }; + insta::assert_json_snapshot!(format!("rss_{name}"), summary, { + ".rss_stat[].pid" => "[pid]", + ".rmap[].pid" => "[pid]", + }); + + if let Some(layout) = raw_report + .lines() + .find_map(|line| line.strip_prefix("Layout:")) + { + let values: Vec = layout + .split_whitespace() + .map(|token| u64::from_str_radix(token.trim_start_matches("0x"), 16)) + .collect::>()?; + let [base, hole_off, hole_len, len] = values[..] else { + panic!("malformed Layout line: {layout}"); + }; + assert_rmap_hole_addresses(&events, base, hole_off, hole_len, len); + } + + // ThpKb > 0 means the MADV_HUGEPAGE region really faulted as PMD folios, so + // huge-folio accounting must be visible: a +512-page delta from the new-anon + // fault path and a -512-page delta that can only come from the + // folio_remove_rmap_pmd hook (MADV_DONTNEED / munmap of a pmd-mapped THP). + if let Some(thp) = raw_report + .lines() + .find_map(|line| line.strip_prefix("ThpKb:")) + { + let thp_kb = u64::from_str_radix(thp.trim().trim_start_matches("0x"), 16)?; + if thp_kb > 0 { + let deltas = events.iter().filter_map(|e| match e.kind { + MemtrackEventKind::Rmap { delta, .. } => Some(delta), + _ => None, + }); + let (mut huge_add, mut huge_remove) = (false, false); + for delta in deltas { + huge_add |= delta >= 512; + huge_remove |= delta <= -512; + } + assert!( + huge_add, + "THP present ({thp_kb} kB) but no huge-folio rmap add" + ); + assert!( + huge_remove, + "THP present ({thp_kb} kB) but no pmd-sized rmap remove" + ); + } + } + Ok(()) +} + +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rss_external_reclaim() -> Result<(), Box> { + let temp_dir = TempDir::new()?; + let binary = shared::compile_c_source( + include_str!("../testdata/rss/madvise_extern.c"), + "madvise_extern", + temp_dir.path(), + )?; + let (events, handle) = shared::track_command(Command::new(&binary))?; + handle.join().unwrap(); + + // Single fork: parent_pid == A (owner), event.pid == B (external caller, single-threaded + // so its tid == its pid). + let (a, b) = events + .iter() + .find_map(|e| match e.kind { + MemtrackEventKind::Fork { parent_pid } => Some((parent_pid, e.pid)), + _ => None, + }) + .expect("expected a fork event"); + + let peak = events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Rss { member: 0, size } if e.pid == a => Some(size), + _ => None, + }) + .max() + .unwrap_or(0); + assert!(peak >= 32 * MIB, "peak file RSS too small: {peak}"); + + // A file decrement owned by A but emitted from B's context: only present when + // out-of-context rss_stat updates are attributed to the owning process. + let external_decrement = events.iter().any(|e| { + e.pid == a + && e.tid == b + && matches!(e.kind, MemtrackEventKind::Rss { member: 0, size } if size < peak) + }); + assert!( + external_decrement, + "external file-RSS decrement not attributed to A (tid=B)" + ); + Ok(()) +} + +/// TEMPORARY diagnostic: track a real-world workload (`ls /nix/store`, ~50 MiB +/// peak on a populated store) and cross-check the reconstructed rss_stat and +/// rmap peaks against the kernel's own accounting (`wait4` ru_maxrss) from an +/// identical untracked run. `ls` is single-process, so raw byte peaks are +/// accumulated directly without per-pid splitting. +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rss_ls_nix_store() -> Result<(), Box> { + if !std::path::Path::new("/nix/store").is_dir() { + eprintln!("skipping: /nix/store not available"); + return Ok(()); + } + + let ls_command = || { + let mut cmd = Command::new("ls"); + cmd.arg("/nix/store").stdout(std::process::Stdio::null()); + cmd + }; + + // Ground truth: identical untracked run, reaped via wait4 for ru_maxrss. + let child = ls_command().spawn()?; + let pid = child.id() as i32; + let mut status = 0i32; + let mut rusage: libc::rusage = unsafe { std::mem::zeroed() }; + let reaped = unsafe { libc::wait4(pid, &mut status, 0, &mut rusage) }; + assert_eq!(reaped, pid, "wait4 failed"); + assert!( + libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0, + "untracked ls failed: status {status}" + ); + let truth_bytes = rusage.ru_maxrss as u64 * 1024; + + let (events, handle) = shared::track_command_with_rmap(ls_command())?; + handle.join().unwrap(); + + let mut rss = RssAccum::default(); + let mut rmap = RmapAccum::default(); + for event in events.iter().sorted_by_key(|event| event.timestamp) { + match event.kind { + MemtrackEventKind::Rss { member, size } => { + if let Ok(index @ 0..4) = usize::try_from(member) { + rss.latest[index] = size; + rss.update_peaks(); + } + } + MemtrackEventKind::Rmap { member, delta } => { + if let Ok(index @ 0..4) = usize::try_from(member) { + rmap.totals[index] += delta * 4096; + rmap.update_peaks(); + } + } + _ => {} + } + } + + let rss_bytes = rss.max_rss; + let rmap_bytes = rmap.max_rss.max(0) as u64; + eprintln!( + "ls /nix/store max RSS: wait4={:.1} MiB rss_stat={:.1} MiB rmap={:.1} MiB", + truth_bytes as f64 / MIB as f64, + rss_bytes as f64 / MIB as f64, + rmap_bytes as f64 / MIB as f64, + ); + + let within = |measured: u64| { + (truth_bytes as f64 * 0.8..=truth_bytes as f64 * 1.2).contains(&(measured as f64)) + }; + assert!( + within(rss_bytes), + "rss_stat peak {rss_bytes} outside 20% of wait4 {truth_bytes}" + ); + assert!( + within(rmap_bytes), + "rmap peak {rmap_bytes} outside 20% of wait4 {truth_bytes}" + ); + Ok(()) +} + +/// rss_stat is an absolute kernel counter; the rmap estimate is reconstructed +/// from zero by summing folio add/remove deltas. When tracking is enabled only +/// after a process has already faulted a resident region, rss_stat's first +/// reading includes that region but the rmap accumulator never saw its adds, so +/// rmap sits a fixed offset (the pre-enable resident set) below rss_stat. +/// +/// The fixture faults a 64 MiB anon baseline before `enable_tracking`, then a +/// 64 MiB anon region after. Reduced with the same Exec-reset lifecycle the +/// production parser uses, rss_stat peaks at ~128 MiB (absolute) while rmap +/// peaks at ~64 MiB (post-enable growth only). +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rss_rmap_late_enable_baseline_loss() -> Result<(), Box> { + const REGION_MIB: u64 = 64; + + let temp_dir = TempDir::new()?; + std::fs::write( + temp_dir.path().join("rss_report.h"), + include_str!("../testdata/rss/rss_report.h"), + )?; + let binary = shared::compile_c_source( + include_str!("../testdata/rss/rmap_late_enable.c"), + "rmap_late_enable", + temp_dir.path(), + )?; + let report_path = temp_dir.path().join("rmap_late_enable.report"); + let ready_path = temp_dir.path().join("ready"); + let go_path = temp_dir.path().join("go"); + + let mut command = Command::new(&binary); + command.arg(&report_path).arg(&ready_path).arg(&go_path); + + let (events, handle) = + shared::track_command_with_rmap_late_enable(command, &ready_path, &go_path)?; + handle.join().unwrap(); + + let (rss_stat, rmap) = per_pid_peaks(&events); + let rss = rss_stat.first().ok_or("no rss_stat pid observed")?; + let rmap = rmap.first().ok_or("no rmap pid observed")?; + eprintln!( + "late-enable anon peaks: rss_stat={} MiB rmap={} MiB (region={} MiB each)", + rss.anon_mib, rmap.anon_mib, REGION_MIB + ); + + // rss_stat's absolute counter covers baseline + growth. + assert!( + rss.anon_mib >= 2 * REGION_MIB - 16, + "rss_stat anon peak {} MiB below the expected ~{} MiB baseline+growth", + rss.anon_mib, + 2 * REGION_MIB + ); + // Post-enable growth is visible to rmap. + assert!( + rmap.anon_mib >= REGION_MIB - 16, + "rmap anon peak {} MiB too small; post-enable growth should be tracked", + rmap.anon_mib + ); + // The reproduction: rmap misses the pre-enable baseline, undercounting + // rss_stat by roughly one region. + let gap = rss.anon_mib.saturating_sub(rmap.anon_mib); + assert!( + gap >= REGION_MIB - 16, + "expected rmap to undercount rss_stat by ~{} MiB (pre-enable baseline loss); gap was {} MiB", + REGION_MIB, + gap + ); + Ok(()) +} diff --git a/crates/memtrack/tests/shared.rs b/crates/memtrack/tests/shared.rs index d31bce09..9d5a040a 100644 --- a/crates/memtrack/tests/shared.rs +++ b/crates/memtrack/tests/shared.rs @@ -6,7 +6,7 @@ use runner_shared::artifacts::{MemtrackEvent as Event, MemtrackEventKind}; use std::path::Path; use std::process::Command; -type TrackResult = anyhow::Result<(Vec, std::thread::JoinHandle<()>)>; +pub type TrackResult = anyhow::Result<(Vec, std::thread::JoinHandle<()>)>; /// Asserts memory events using snapshot testing without marker filtering. /// @@ -27,6 +27,17 @@ macro_rules! assert_events_snapshot { // Dedup events by address and type to remove duplicates let events = $events .iter() + .filter(|e| { + // Allocation snapshots track only allocator events; RSS and + // process-lifecycle events are asserted by dedicated tests. + !matches!( + e.kind, + MemtrackEventKind::Rss { .. } + | MemtrackEventKind::Fork { .. } + | MemtrackEventKind::Exec + | MemtrackEventKind::Exit + ) + }) .sorted_by_key(|e| e.timestamp) .dedup_by(|a, b| a.addr == b.addr && discriminant(&a.kind) == discriminant(&b.kind)) .collect::>(); @@ -76,6 +87,7 @@ macro_rules! assert_events_with_marker { // Remove events outside our 0xC0D59EED marker allocations let filtered_events = $events .iter() + .filter(|e| !matches!(e.kind, MemtrackEventKind::Rss { .. })) .sorted_by_key(|e| e.timestamp) .dedup_by(|a, b| a.addr == b.addr && discriminant(&a.kind) == discriminant(&b.kind)) .skip_while(|e| { @@ -141,12 +153,46 @@ pub fn track_binary(binary: &Path) -> TrackResult { track_command(Command::new(binary)) } +pub fn compile_c_source( + source_code: &str, + name: &str, + output_dir: &Path, +) -> Result> { + let source_path = output_dir.join(format!("{name}.c")); + let binary_path = output_dir.join(name); + std::fs::write(&source_path, source_code)?; + + let output = Command::new("gcc") + .args(["-O0", "-o", binary_path.to_str().unwrap()]) + .arg(&source_path) + .output()?; + if !output.status.success() { + error!("gcc stderr: {}", String::from_utf8_lossy(&output.stderr)); + return Err("Failed to compile C fixture".into()); + } + + Ok(binary_path) +} + /// Track a command, collecting all memory events. /// /// No allocators are pre-attached: the exec-mapping watcher discovers and /// attaches them as the tracked tree maps executable files. pub fn track_command(command: Command) -> TrackResult { - let tracker = Tracker::new()?; + track_command_impl(command, false) +} + +/// Track a command with folio rmap hooks enabled, reconstructing per-process RSS. +pub fn track_command_with_rmap(command: Command) -> TrackResult { + track_command_impl(command, true) +} + +fn track_command_impl(command: Command, track_rmap: bool) -> TrackResult { + let tracker = if track_rmap { + Tracker::new_without_allocators_with_rmap(true)? + } else { + Tracker::new()? + }; tracker.enable_tracking()?; let mut session = tracker.spawn(&command, None)?; @@ -168,3 +214,50 @@ pub fn track_command(command: Command) -> TrackResult { Ok((events, thread_handle)) } + +/// Track a command with rmap, enabling tracking only after the target creates +/// `ready_path`. The target is spawned and resumed first, so any memory it +/// faults before signalling `ready` is already resident when tracking turns on. +/// The caller enables tracking, then creates `go_path` to release the target. +pub fn track_command_with_rmap_late_enable( + command: Command, + ready_path: &Path, + go_path: &Path, +) -> TrackResult { + let tracker = Tracker::new_without_allocators_with_rmap(true)?; + + let mut session = tracker.spawn(&command, None)?; + let rx = session.take_events()?; + + let handshake = (|| -> anyhow::Result<()> { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while !ready_path.exists() { + if std::time::Instant::now() > deadline { + anyhow::bail!("target never signalled baseline-ready"); + } + std::thread::sleep(std::time::Duration::from_millis(2)); + } + tracker.enable_tracking()?; + std::fs::write(go_path, b"go")?; + Ok(()) + })(); + + // A failed handshake leaves the target blocked on `go_path`; Session has no + // Drop kill, so reap it explicitly before propagating or the test hangs. + if let Err(e) = handshake { + unsafe { libc::kill(session.pid(), libc::SIGKILL) }; + let _ = session.wait(); + let _ = tracker.finish(); + return Err(e); + } + + session.wait()?; + drop(session); + let events: Vec = rx.iter().collect(); + + tracker.finish()?; + let thread_handle = std::thread::spawn(move || drop(tracker)); + + info!("Tracked {} events (late enable)", events.len()); + Ok((events, thread_handle)) +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_anon.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_anon.snap new file mode 100644 index 00000000..bd39e6cc --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_anon.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 64, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_file.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_file.snap new file mode 100644 index 00000000..2745b74d --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_file.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 0, + "RssFile:": 64, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 64, + "anon_mib": 0, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 64, + "anon_mib": 0, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_fork.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_fork.snap new file mode 100644 index 00000000..bc24de5b --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_fork.snap @@ -0,0 +1,42 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "ChildRssAnonKb:": 64, + "ParentRssAnonKb:": 32 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + }, + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + }, + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_fork_idle.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_fork_idle.snap new file mode 100644 index 00000000..3a97ec7d --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_fork_idle.snap @@ -0,0 +1,45 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "ChildRssAnon:": 32, + "MaxRssKb:": 32, + "RssAnon:": 32, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + }, + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + }, + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_madvise.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_madvise.snap new file mode 100644 index 00000000..bd39e6cc --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_madvise.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 64, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_mremap_move.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_mremap_move.snap new file mode 100644 index 00000000..6e2a2202 --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_mremap_move.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 32, + "RssAnon:": 32, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_munmap_hole.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_munmap_hole.snap new file mode 100644 index 00000000..bd39e6cc --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_munmap_hole.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 64, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_shmem.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_shmem.snap new file mode 100644 index 00000000..19bb441e --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_shmem.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 0, + "RssFile:": 0, + "RssShmem:": 64 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 0, + "shmem_mib": 64, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 0, + "shmem_mib": 64, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_triangle.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_triangle.snap new file mode 100644 index 00000000..bd39e6cc --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_triangle.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 64, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} From 8bc57e45f6fdf86e6b9bc72c294b5332ee65f053 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 21 Jul 2026 16:36:04 +0200 Subject: [PATCH 06/13] feat(memtrack): attribute foreign-actor rmap events via mm ownership Recover the owning pid for rmap events run by another task (reclaim, process_madvise, khugepaged, KSM) from the mm_struct pointer, and maintain the ownership maps across exec and thread-group exit. --- crates/memtrack/src/ebpf/c/rss.bpf.h | 141 ++++++++++++- .../memtrack/src/ebpf/c/utils/event_helpers.h | 34 ++- .../src/ebpf/c/utils/process_tracking.h | 12 -- crates/memtrack/src/ebpf/memtrack/tracking.rs | 2 - crates/memtrack/src/ebpf/tracker.rs | 6 +- crates/memtrack/src/main.rs | 9 +- .../memtrack/testdata/rss/rmap_leader_exit.c | 74 +++++++ .../memtrack/testdata/rss/rmap_thread_fork.c | 60 ++++++ crates/memtrack/testdata/rss/rss_report.h | 18 +- crates/memtrack/tests/rss_tests.rs | 197 ++++++++++++++++-- 10 files changed, 491 insertions(+), 62 deletions(-) create mode 100644 crates/memtrack/testdata/rss/rmap_leader_exit.c create mode 100644 crates/memtrack/testdata/rss/rmap_thread_fork.c diff --git a/crates/memtrack/src/ebpf/c/rss.bpf.h b/crates/memtrack/src/ebpf/c/rss.bpf.h index ca208bf3..2850a634 100644 --- a/crates/memtrack/src/ebpf/c/rss.bpf.h +++ b/crates/memtrack/src/ebpf/c/rss.bpf.h @@ -22,6 +22,30 @@ struct { __type(value, struct rss_owner); } mm_to_pid SEC(".maps"); +/* Foreign-actor rmap attribution: rmap events run by a task other than the mm's + * owner (kswapd reclaim, another process's process_madvise, khugepaged, KSM, + * uffd) carry no owning-pid context, so mm_owner recovers it from the mm_struct + * pointer. pid_mm is the inverse, letting the exec and exit hooks remove an entry + * by value. + * + * Lifecycle invariant: every mm_owner entry is removed when its process execs + * (the old mm is freed mid-life) or when its thread group dies, whichever comes + * first; LRU eviction is only a backstop. A stale entry surviving mm-pointer + * reuse would misattribute another process's events, so ownership is only ever + * registered from an in-context (task->mm == mm) event. + * + * pid_mm is a plain hash on purpose: an LRU inverse could be evicted while its + * forward twin stays lookup-hot, leaving exec/exit unable to remove the live + * mm_owner entry. Like tracked_pids, its entries are bound to the process + * lifecycle and removed at group death. */ +struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __uint(max_entries, 10240); + __type(key, __u64); + __type(value, __u32); +} mm_owner SEC(".maps"); +BPF_HASH_MAP(pid_mm, __u32, __u64, 10240); + #define FOLIO_MAPPING_ANON 0x1UL const volatile __u32 page_shift = 12; @@ -144,17 +168,50 @@ static __always_inline int submit_rmap(struct vm_area_struct* vma, __s32 member, __u64 addr) { __u64 mm = (__u64)BPF_CORE_READ(vma, vm_mm); struct task_struct* task = bpf_get_current_task_btf(); - if ((__u64)BPF_CORE_READ(task, mm) != mm) { - return 0; - } - __u64 tid = bpf_get_current_pid_tgid(); __u32 pid = tid >> 32; - if (!is_tracked(pid)) { + + if ((__u64)BPF_CORE_READ(task, mm) == mm) { + if (!is_tracked(pid)) { + return 0; + } + + /* Register ownership so foreign actors can later attribute to this pid. + * Both maps are validated (not just written) on every in-context event: + * the lookups keep the hot path cheap AND keep both entries LRU-fresh, + * since pid_mm is otherwise never read until exec/exit and could be + * evicted independently of its still-hot mm_owner twin. */ + __u32* owner = bpf_map_lookup_elem(&mm_owner, &mm); + if (!owner || *owner != pid) { + bpf_map_update_elem(&mm_owner, &mm, &pid, BPF_ANY); + } + __u64* cur_mm = bpf_map_lookup_elem(&pid_mm, &pid); + if (!cur_mm || *cur_mm != mm) { + bpf_map_update_elem(&pid_mm, &pid, &mm, BPF_ANY); + } + + SUBMIT_EVENT_AS(pid, EVENT_TYPE_RMAP, { + e->data.rmap.member = member; + e->data.rmap.delta = delta; + e->data.rmap.addr = addr; + }); + } + + /* Foreign actor (task->mm != mm, including kthreads whose task->mm is NULL): + * recover the owner from the in-context registration. Fail toward dropping + * the event on any uncertainty about ownership. */ + __u32* found = bpf_map_lookup_elem(&mm_owner, &mm); + if (!found) { + return 0; + } + __u32 owner = *found; + if (!is_tracked(owner)) { return 0; } - SUBMIT_EVENT_AS(pid, EVENT_TYPE_RMAP, { + /* SUBMIT_EVENT_AS stamps header.tid from the current task, identifying the + * foreign actor that performed the rmap change. */ + SUBMIT_EVENT_AS(owner, EVENT_TYPE_RMAP, { e->data.rmap.member = member; e->data.rmap.delta = delta; e->data.rmap.addr = addr; @@ -245,23 +302,87 @@ int tracepoint_task_newtask(struct trace_event_raw_task_newtask* ctx) { return 0; } + /* Register the child here rather than on sched_process_fork: that + * tracepoint fires for CLONE_THREAD too and carries only raw task pids, + * which would fill the tracking maps with thread tids that no exit path + * removes (group death deletes only the tgid). task_newtask fires before + * wake_up_new_task, so registration precedes any event from the child. */ __u32 child_pid = ctx->pid; + track_child(child_pid, parent_pid); + SUBMIT_EVENT_AS(child_pid, EVENT_TYPE_FORK, { e->data.fork.parent_pid = parent_pid; }); } +/* Remove pid's ownership registration. The mm_owner value is verified against + * pid before deleting: a stale pid_mm entry (LRU eviction skew) could otherwise + * point at an mm since re-registered by another process, and deleting that + * would silence a live owner's foreign attribution. */ +static __always_inline void drop_mm_ownership(__u32 pid) { + __u64* mm = bpf_map_lookup_elem(&pid_mm, &pid); + if (mm) { + __u32* owner = bpf_map_lookup_elem(&mm_owner, mm); + if (owner && *owner == pid) { + bpf_map_delete_elem(&mm_owner, mm); + } + } + bpf_map_delete_elem(&pid_mm, &pid); +} + SEC("tracepoint/sched/sched_process_exec") int tracepoint_sched_process_exec(void* ctx) { - SUBMIT_EVENT(EVENT_TYPE_EXEC, {}); + __u32 pid = bpf_get_current_pid_tgid() >> 32; + if (!is_tracked(pid)) { + return 0; + } + + /* Maintain ownership before submitting (SUBMIT_EVENT_AS returns from the + * function). Exec frees the old mm long before group death, so the stale + * pointer must be dropped here or a reused mm_struct would be misattributed. */ + drop_mm_ownership(pid); + + struct task_struct* task = bpf_get_current_task_btf(); + __u64 new_mm = (__u64)BPF_CORE_READ(task, mm); + if (new_mm) { + bpf_map_update_elem(&mm_owner, &new_mm, &pid, BPF_ANY); + bpf_map_update_elem(&pid_mm, &pid, &new_mm, BPF_ANY); + } + + SUBMIT_EVENT_AS(pid, EVENT_TYPE_EXEC, {}); } SEC("tracepoint/sched/sched_process_exit") int tracepoint_sched_process_exit(void* ctx) { - __u64 tid = bpf_get_current_pid_tgid(); - if ((tid >> 32) != (tid & 0xFFFFFFFF)) { + __u32 pid = bpf_get_current_pid_tgid() >> 32; + if (!is_tracked(pid)) { return 0; } - SUBMIT_EVENT(EVENT_TYPE_EXIT, {}); + /* EXIT marks the death of the whole thread group, not of one thread: the + * leader can pthread_exit while workers keep running, and the last thread + * to exit need not be the leader. do_exit decrements signal->live before + * this tracepoint fires, so live == 0 identifies the dying thread group's + * final exit — but concurrently exiting threads can BOTH read 0, so the + * tracked_pids delete below arbitrates: only the task that wins it emits. */ + struct task_struct* task = bpf_get_current_task_btf(); + if (BPF_CORE_READ(task, signal, live.counter) != 0) { + return 0; + } + + /* Untrack the pid before submitting: lifetime events are gated only on + * is_tracked, so a stale entry would keep streaming events if the kernel + * reuses the pid for an unrelated process. Untracking here also keeps the + * fixed-size tracking maps from filling up over long sessions. The delete + * doubles as the exactly-once claim on EXIT. */ + if (bpf_map_delete_elem(&tracked_pids, &pid) != 0) { + return 0; + } + bpf_map_delete_elem(&pids_ppid, &pid); + + /* Drop the ownership mapping so foreign actors stop attributing to a pid + * the kernel may reuse. */ + drop_mm_ownership(pid); + + SUBMIT_EVENT_AS(pid, EVENT_TYPE_EXIT, {}); } #endif /* __RSS_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/c/utils/event_helpers.h b/crates/memtrack/src/ebpf/c/utils/event_helpers.h index 1be39e7e..fbbc0536 100644 --- a/crates/memtrack/src/ebpf/c/utils/event_helpers.h +++ b/crates/memtrack/src/ebpf/c/utils/event_helpers.h @@ -38,14 +38,19 @@ static __always_inline __u64* take_param(void* map) { return value; } +/* Submission is split into two classes: + * - lifetime events (rss_stat, rmap, fork/exec/exit): emitted whenever the + * pid is tracked, ignoring the enable toggle. The parser reconstructs + * absolute per-process state from these, so it needs every event from + * process birth — a delta stream that starts mid-life can never recover + * the resident baseline faulted before enable. + * - allocator events (malloc/free/mmap/...): high-volume and only meaningful + * inside a measurement window, so they stay behind is_enabled(). + */ #define SUBMIT_EVENT_AS(owner_pid, evt_type, fill_data) \ { \ __u64 tid = bpf_get_current_pid_tgid(); \ \ - if (!is_enabled()) { \ - return 0; \ - } \ - \ struct event* e = bpf_ringbuf_reserve(&events, sizeof(*e), 0); \ if (!e) { \ __u32 zero = 0; \ @@ -79,33 +84,42 @@ static __always_inline __u64* take_param(void* map) { SUBMIT_EVENT_AS(pid, evt_type, fill_data); \ } +#define SUBMIT_GATED_EVENT(evt_type, fill_data) \ + { \ + if (!is_enabled()) { \ + return 0; \ + } \ + \ + SUBMIT_EVENT(evt_type, fill_data); \ + } + static __always_inline int submit_alloc_event(__u64 size, __u64 addr) { - SUBMIT_EVENT(EVENT_TYPE_MALLOC, { + SUBMIT_GATED_EVENT(EVENT_TYPE_MALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; }); } static __always_inline int submit_aligned_alloc_event(__u64 size, __u64 addr) { - SUBMIT_EVENT(EVENT_TYPE_ALIGNED_ALLOC, { + SUBMIT_GATED_EVENT(EVENT_TYPE_ALIGNED_ALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; }); } static __always_inline int submit_calloc_event(__u64 size, __u64 addr) { - SUBMIT_EVENT(EVENT_TYPE_CALLOC, { + SUBMIT_GATED_EVENT(EVENT_TYPE_CALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; }); } static __always_inline int submit_free_event(__u64 addr) { - SUBMIT_EVENT(EVENT_TYPE_FREE, { e->data.free.addr = addr; }); + SUBMIT_GATED_EVENT(EVENT_TYPE_FREE, { e->data.free.addr = addr; }); } static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, __u64 size) { - SUBMIT_EVENT(EVENT_TYPE_REALLOC, { + SUBMIT_GATED_EVENT(EVENT_TYPE_REALLOC, { e->data.realloc.old_addr = old_addr; e->data.realloc.new_addr = new_addr; e->data.realloc.size = size; @@ -113,7 +127,7 @@ static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, } static __always_inline int submit_mmap_event(__u64 addr, __u64 size, __u8 event_type) { - SUBMIT_EVENT(event_type, { + SUBMIT_GATED_EVENT(event_type, { e->data.mmap.addr = addr; e->data.mmap.size = size; }); diff --git a/crates/memtrack/src/ebpf/c/utils/process_tracking.h b/crates/memtrack/src/ebpf/c/utils/process_tracking.h index 19cff338..69dd2a73 100644 --- a/crates/memtrack/src/ebpf/c/utils/process_tracking.h +++ b/crates/memtrack/src/ebpf/c/utils/process_tracking.h @@ -43,16 +43,4 @@ static __always_inline void track_child(__u32 child_pid, __u32 parent_pid) { bpf_map_update_elem(&pids_ppid, &child_pid, &parent_pid, BPF_ANY); } -SEC("tracepoint/sched/sched_process_fork") -int tracepoint_sched_fork(struct trace_event_raw_sched_process_fork* ctx) { - __u32 parent_pid = ctx->parent_pid; - __u32 child_pid = ctx->child_pid; - - if (is_tracked(parent_pid)) { - track_child(child_pid, parent_pid); - } - - return 0; -} - #endif /* __PROCESS_TRACKING_H__ */ diff --git a/crates/memtrack/src/ebpf/memtrack/tracking.rs b/crates/memtrack/src/ebpf/memtrack/tracking.rs index bc8156c4..38e11a93 100644 --- a/crates/memtrack/src/ebpf/memtrack/tracking.rs +++ b/crates/memtrack/src/ebpf/memtrack/tracking.rs @@ -3,7 +3,6 @@ use crate::prelude::*; use paste::paste; impl MemtrackBpf { - attach_tracepoint!(sched_fork); attach_tracepoint!(rss_stat); attach_tracepoint!(task_newtask); attach_tracepoint!(sched_process_exec); @@ -50,7 +49,6 @@ impl MemtrackBpf { } pub fn attach_tracepoints(&mut self) -> Result<()> { - self.attach_sched_fork()?; self.attach_task_newtask()?; self.attach_sched_process_exec()?; self.attach_sched_process_exit()?; diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index ff6c0881..83b34853 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -79,12 +79,14 @@ impl Tracker { Ok(Session::new(child, rx, poller)) } - /// Enable event tracking in the BPF program + /// Enable allocator-event tracking in the BPF program. Lifetime events + /// (rss_stat, rmap, fork/exec/exit) are emitted for tracked pids + /// regardless of this toggle. pub fn enable_tracking(&self) -> Result<()> { self.bpf.lock().enable_tracking() } - /// Disable event tracking in the BPF program + /// Disable allocator-event tracking in the BPF program pub fn disable_tracking(&self) -> Result<()> { self.bpf.lock().disable_tracking() } diff --git a/crates/memtrack/src/main.rs b/crates/memtrack/src/main.rs index 8ad48791..283cff19 100644 --- a/crates/memtrack/src/main.rs +++ b/crates/memtrack/src/main.rs @@ -95,8 +95,9 @@ fn track_command( } })) } else { - // Without IPC, nothing toggles the tracking_enabled map, so events would - // be dropped by the eBPF is_enabled() check. Enable it up front. + // Without IPC, nothing toggles the tracking_enabled map, so allocator + // events would be dropped by the eBPF is_enabled() check. Enable it up + // front. tracker.enable_tracking()?; None }; @@ -135,8 +136,8 @@ fn track_command( let status = session.wait().context("Failed to wait for command")?; debug!("Command exited with status: {status}"); - // Stop event production before draining: the child has exited, so anything - // still arriving is already in the ring buffer. + // Stop allocator-event production before draining: the child has exited, + // so anything still arriving is already in the ring buffer. if let Err(e) = tracker.disable_tracking() { warn!("Failed to disable tracking: {e:#}"); } diff --git a/crates/memtrack/testdata/rss/rmap_leader_exit.c b/crates/memtrack/testdata/rss/rmap_leader_exit.c new file mode 100644 index 00000000..51e7b23c --- /dev/null +++ b/crates/memtrack/testdata/rss/rmap_leader_exit.c @@ -0,0 +1,74 @@ +#include +#include +#include +#include +#include +#include + +#include "rss_report.h" + +/* The main thread publishes its tid and leaves via pthread_exit while a worker + * thread keeps the process alive. The worker waits until the leader is a + * zombie — its exit path, including the sched_process_exit tracepoint, has + * fully run — then faults an anon region, writes the report, and exits the + * whole group. A tracker keyed on leader exit instead of thread-group death + * would stop watching before the worker's region is faulted. + * + * argv: [1]=report path. */ + +static const char* report_path; +static pid_t leader_tid; + +static int leader_is_zombie(void) { + char path[64]; + char buf[256]; + snprintf(path, sizeof(path), "/proc/self/task/%d/stat", leader_tid); + FILE* f = fopen(path, "r"); + if (!f) { + return 1; + } + size_t n = fread(buf, 1, sizeof(buf) - 1, f); + fclose(f); + buf[n] = '\0'; + /* The state field follows the parenthesized comm. */ + const char* p = strrchr(buf, ')'); + if (!p || p[1] == '\0' || p[2] == '\0') { + return 0; + } + return p[2] == 'Z'; +} + +static void* worker(void* arg) { + (void)arg; + while (!leader_is_zombie()) { + usleep(1000); + } + + size_t region = 64UL * 1024 * 1024; + void* mem = + mmap(NULL, region, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) { + _exit(1); + } + memset(mem, 0x42, region); + + /* The leader is a zombie by now, so /proc//status has no Rss lines; + * report through this worker's own task instead. */ + int ret = write_rss_report_pid((int)syscall(SYS_gettid), report_path); + munmap(mem, region); + _exit(ret); +} + +int main(int argc, char** argv) { + if (argc != 2) { + return 1; + } + report_path = argv[1]; + leader_tid = (pid_t)syscall(SYS_gettid); + + pthread_t thread; + if (pthread_create(&thread, NULL, worker, NULL) != 0) { + return 1; + } + pthread_exit(NULL); +} diff --git a/crates/memtrack/testdata/rss/rmap_thread_fork.c b/crates/memtrack/testdata/rss/rmap_thread_fork.c new file mode 100644 index 00000000..831fd110 --- /dev/null +++ b/crates/memtrack/testdata/rss/rmap_thread_fork.c @@ -0,0 +1,60 @@ +#include +#include +#include +#include +#include + +#include "rss_report.h" + +/* A worker thread (not the group leader) calls fork(); the child faults an + * anon region and writes the report. Child tracking must key on the parent's + * TGID: a scheme keyed on the raw creator tid would only cover forks issued + * by the leader. + * + * argv: [1]=report path. */ + +static const char* report_path; + +static void* worker(void* arg) { + (void)arg; + + pid_t pid = fork(); + if (pid < 0) { + _exit(1); + } + if (pid == 0) { + size_t region = 64UL * 1024 * 1024; + void* mem = mmap(NULL, region, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) { + _exit(1); + } + memset(mem, 0x42, region); + + int ret = write_rss_report(report_path); + munmap(mem, region); + _exit(ret); + } + + int status; + if (waitpid(pid, &status, 0) < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) { + _exit(1); + } + return NULL; +} + +int main(int argc, char** argv) { + if (argc != 2) { + return 1; + } + report_path = argv[1]; + + pthread_t thread; + if (pthread_create(&thread, NULL, worker, NULL) != 0) { + return 1; + } + if (pthread_join(thread, NULL) != 0) { + return 1; + } + return 0; +} diff --git a/crates/memtrack/testdata/rss/rss_report.h b/crates/memtrack/testdata/rss/rss_report.h index e144d145..169bdc07 100644 --- a/crates/memtrack/testdata/rss/rss_report.h +++ b/crates/memtrack/testdata/rss/rss_report.h @@ -26,14 +26,18 @@ static long rss_status_kb(const char* key) { return rss_status_kb_pid(getpid(), key); } -static int write_rss_report(const char* path) { - long anon = rss_status_kb("RssAnon:"); - long file = rss_status_kb("RssFile:"); - long shmem = rss_status_kb("RssShmem:"); +/* Reads Rss* through /proc//status of the given task. For the + * thread-group dir this is the leader's task: once the leader is a zombie its + * mm pointer is gone and the Rss lines disappear, so callers whose leader has + * exited must pass a live thread's tid instead. */ +static int write_rss_report_pid(int pid, const char* path) { + long anon = rss_status_kb_pid(pid, "RssAnon:"); + long file = rss_status_kb_pid(pid, "RssFile:"); + long shmem = rss_status_kb_pid(pid, "RssShmem:"); /* VmHWM instead of getrusage(): ru_maxrss includes signal->maxrss, which * survives execve and so reports the peak of the pre-exec parent image. * VmHWM belongs to the mm and starts fresh at exec. */ - long max_rss = rss_status_kb("VmHWM:"); + long max_rss = rss_status_kb_pid(pid, "VmHWM:"); if (anon < 0 || file < 0 || shmem < 0 || max_rss < 0) { return 1; } @@ -47,4 +51,8 @@ static int write_rss_report(const char* path) { return 0; } +static int write_rss_report(const char* path) { + return write_rss_report_pid(getpid(), path); +} + #endif diff --git a/crates/memtrack/tests/rss_tests.rs b/crates/memtrack/tests/rss_tests.rs index 8875096b..41a044a4 100644 --- a/crates/memtrack/tests/rss_tests.rs +++ b/crates/memtrack/tests/rss_tests.rs @@ -199,6 +199,11 @@ fn first_fork_pair(events: &[MemtrackEvent]) -> Option<(i32, i32)> { /// Pins reconstructed rmap addresses to the exact punched range: hole pages are /// the only ones removed and later re-added; every other page in the region is /// added first and only removed afterwards. +/// +/// Only own-context events (tid == pid; the fixture is single-threaded) are +/// considered: foreign actors like kcompactd migrating a page produce a +/// remove-then-add on arbitrary pages, which the attribution of foreign rmap +/// events makes visible here. fn assert_rmap_hole_addresses( events: &[MemtrackEvent], base: u64, @@ -216,6 +221,9 @@ fn assert_rmap_hole_addresses( let MemtrackEventKind::Rmap { delta, .. } = event.kind else { continue; }; + if event.tid != event.pid { + continue; + } if event.addr < base || event.addr >= base + len { continue; } @@ -364,6 +372,68 @@ fn test_rss_external_reclaim() -> Result<(), Box> { Ok(()) } +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rss_rmap_external_reclaim() -> Result<(), Box> { + let temp_dir = TempDir::new()?; + let binary = shared::compile_c_source( + include_str!("../testdata/rss/madvise_extern.c"), + "madvise_extern", + temp_dir.path(), + )?; + let (events, handle) = shared::track_command_with_rmap(Command::new(&binary))?; + handle.join().unwrap(); + + // Single fork: parent_pid == A (owner that faulted the file region), event.pid == B + // (external caller of process_madvise(MADV_PAGEOUT) against A from its own context). + let (a, b) = events + .iter() + .find_map(|e| match e.kind { + MemtrackEventKind::Fork { parent_pid } => Some((parent_pid, e.pid)), + _ => None, + }) + .expect("expected a fork event"); + + // Sanity: A's own in-context file-page faults are reconstructed by rmap. + let in_context_add: i64 = events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Rmap { member: 0, delta } if e.pid == a && delta > 0 => Some(delta), + _ => None, + }) + .sum(); + let in_context_bytes = in_context_add as u64 * 4096; + assert!( + in_context_bytes >= 32 * MIB, + "in-context file rmap adds too small: {in_context_bytes}" + ); + + // The point of the test: a file-page remove owned by A but emitted from B's + // context (tid == B), only present when the foreign reclaim's rmap events are + // attributed to the owning process via the mm_owner map. + let external_removed: i64 = events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Rmap { member: 0, delta } + if e.pid == a && e.tid == b && delta < 0 => + { + Some(-delta) + } + _ => None, + }) + .sum(); + assert!( + external_removed > 0, + "external MADV_PAGEOUT remove not attributed to the owner (pid=A, tid=B)" + ); + let external_bytes = external_removed as u64 * 4096; + assert!( + external_bytes >= 8 * MIB, + "external MADV_PAGEOUT remove not attributed to the owner: only {external_bytes} bytes" + ); + Ok(()) +} + /// TEMPORARY diagnostic: track a real-world workload (`ls /nix/store`, ~50 MiB /// peak on a populated store) and cross-check the reconstructed rss_stat and /// rmap peaks against the kernel's own accounting (`wait4` ru_maxrss) from an @@ -443,18 +513,18 @@ fn test_rss_ls_nix_store() -> Result<(), Box> { } /// rss_stat is an absolute kernel counter; the rmap estimate is reconstructed -/// from zero by summing folio add/remove deltas. When tracking is enabled only -/// after a process has already faulted a resident region, rss_stat's first -/// reading includes that region but the rmap accumulator never saw its adds, so -/// rmap sits a fixed offset (the pre-enable resident set) below rss_stat. +/// from zero by summing folio add/remove deltas. Both are emitted for the +/// whole lifetime of a tracked pid, independent of the enable toggle (which +/// only gates allocator events): a delta stream that starts mid-life could +/// never recover the resident baseline faulted before enable. /// /// The fixture faults a 64 MiB anon baseline before `enable_tracking`, then a /// 64 MiB anon region after. Reduced with the same Exec-reset lifecycle the -/// production parser uses, rss_stat peaks at ~128 MiB (absolute) while rmap -/// peaks at ~64 MiB (post-enable growth only). +/// production parser uses, both series peak at ~128 MiB: the pre-enable +/// baseline is visible to rmap because the pid is tracked from spawn. #[test_with::env(GITHUB_ACTIONS)] #[test] -fn test_rss_rmap_late_enable_baseline_loss() -> Result<(), Box> { +fn test_rss_rmap_late_enable_covers_baseline() -> Result<(), Box> { const REGION_MIB: u64 = 64; let temp_dir = TempDir::new()?; @@ -493,20 +563,113 @@ fn test_rss_rmap_late_enable_baseline_loss() -> Result<(), Box= REGION_MIB - 16, - "rmap anon peak {} MiB too small; post-enable growth should be tracked", - rmap.anon_mib + rmap.anon_mib >= 2 * REGION_MIB - 16, + "rmap anon peak {} MiB misses the pre-enable baseline; expected ~{} MiB", + rmap.anon_mib, + 2 * REGION_MIB ); - // The reproduction: rmap misses the pre-enable baseline, undercounting - // rss_stat by roughly one region. - let gap = rss.anon_mib.saturating_sub(rmap.anon_mib); + let gap = rss.anon_mib.abs_diff(rmap.anon_mib); assert!( - gap >= REGION_MIB - 16, - "expected rmap to undercount rss_stat by ~{} MiB (pre-enable baseline loss); gap was {} MiB", - REGION_MIB, + gap <= 16, + "rss_stat ({} MiB) and rmap ({} MiB) diverged by {} MiB despite lifetime tracking", + rss.anon_mib, + rmap.anon_mib, gap ); Ok(()) } + +/// The leader thread can pthread_exit while worker threads keep the process +/// alive; EXIT must mark thread-group death, not leader exit. The fixture's +/// worker faults a 64 MiB anon region only after the leader is a zombie, so +/// an exit path keyed on the leader would untrack the pid before the region +/// is faulted and emit EXIT ahead of the worker's rmap events. +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rss_rmap_leader_exit_keeps_tracking() -> Result<(), Box> { + const REGION_MIB: u64 = 64; + + let (_raw_report, events) = track_fixture( + include_str!("../testdata/rss/rmap_leader_exit.c"), + "rmap_leader_exit", + shared::track_command_with_rmap, + )?; + + // The fixture process is the pid with the largest anon rmap peak. + let (_rss_stat, rmap) = per_pid_peaks(&events); + let rmap_peak = rmap + .iter() + .max_by_key(|p| p.anon_mib) + .ok_or("no rmap pid observed")?; + assert!( + rmap_peak.anon_mib >= REGION_MIB - 16, + "rmap anon peak {} MiB misses the worker's post-leader-exit region (~{} MiB)", + rmap_peak.anon_mib, + REGION_MIB + ); + + let pid = rmap_peak.pid; + let exits: Vec<&MemtrackEvent> = events + .iter() + .filter(|e| e.pid == pid && matches!(e.kind, MemtrackEventKind::Exit)) + .collect(); + assert_eq!(exits.len(), 1, "expected exactly one EXIT for pid {pid}"); + + let last_rmap_ts = events + .iter() + .filter(|e| e.pid == pid && matches!(e.kind, MemtrackEventKind::Rmap { .. })) + .map(|e| e.timestamp) + .max() + .ok_or("no rmap events for fixture pid")?; + assert!( + exits[0].timestamp > last_rmap_ts, + "EXIT fired before the worker's rmap events: leader exit was treated as process death" + ); + Ok(()) +} + +/// A fork issued by a worker thread must still track the child: registration +/// keys on the parent's tgid (task_newtask fires in the cloning task, whose +/// pid_tgid upper half is the tgid), not on the raw creator tid. +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rss_rmap_thread_fork_tracks_child() -> Result<(), Box> { + const REGION_MIB: u64 = 64; + + let (_raw_report, events) = track_fixture( + include_str!("../testdata/rss/rmap_thread_fork.c"), + "rmap_thread_fork", + shared::track_command_with_rmap, + )?; + + // Single fork in the fixture: parent = the fixture process (tgid), child = + // the region-faulting process. + let (parent, child) = events + .iter() + .find_map(|e| match e.kind { + MemtrackEventKind::Fork { parent_pid } => Some((parent_pid, e.pid)), + _ => None, + }) + .ok_or("no fork event: worker-thread fork was not tracked")?; + assert_ne!(parent, child); + + let child_anon: i64 = events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Rmap { member: 1, delta } if e.pid == child && delta > 0 => { + Some(delta) + } + _ => None, + }) + .sum(); + assert!( + child_anon * 4096 >= ((REGION_MIB - 16) * MIB) as i64, + "child of a worker-thread fork missed rmap tracking: anon adds {} bytes, expected ~{} MiB", + child_anon * 4096, + REGION_MIB + ); + Ok(()) +} From 538e0192c2288517df7ac632a9bf41219cd76bb1 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 22 Jul 2026 12:06:23 +0200 Subject: [PATCH 07/13] test(memtrack): cover exec and foreign-rmap attribution cases Add the exec_anon and foreign_add cases and extend the rss suite for the mm-ownership attribution and thread-group lifecycle paths. --- .../memtrack/testdata/rss/rmap_late_enable.c | 59 --- .../memtrack/testdata/rss/rmap_leader_exit.c | 74 ---- crates/memtrack/tests/rss_tests.rs | 401 +++++------------- crates/memtrack/tests/shared.rs | 47 -- 4 files changed, 96 insertions(+), 485 deletions(-) delete mode 100644 crates/memtrack/testdata/rss/rmap_late_enable.c delete mode 100644 crates/memtrack/testdata/rss/rmap_leader_exit.c diff --git a/crates/memtrack/testdata/rss/rmap_late_enable.c b/crates/memtrack/testdata/rss/rmap_late_enable.c deleted file mode 100644 index 0bdae37b..00000000 --- a/crates/memtrack/testdata/rss/rmap_late_enable.c +++ /dev/null @@ -1,59 +0,0 @@ -#include -#include -#include -#include - -#include "rss_report.h" - -/* Two-phase fixture that faults anonymous memory before and after a handshake: - * - * 1. Fault a baseline region, then signal `ready` and block on `go`. - * 2. After `go` appears, fault a second region of the same (anon) member. - * - * The handshake lets the caller act between the two phases (e.g. start - * observing only phase 2). Both regions touch MM_ANONPAGES, so an absolute - * counter read after phase 2 covers baseline + growth, while a delta observed - * only from phase 2 covers growth alone. - * - * argv: [1]=report path, [2]=ready path, [3]=go path. */ -static void touch(const char* path) { - FILE* f = fopen(path, "w"); - if (f) { - fclose(f); - } -} - -int main(int argc, char** argv) { - if (argc != 4) { - return 1; - } - const char* report_path = argv[1]; - const char* ready_path = argv[2]; - const char* go_path = argv[3]; - - size_t region = 64UL * 1024 * 1024; - - void* baseline = mmap(NULL, region, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (baseline == MAP_FAILED) { - return 1; - } - memset(baseline, 0x42, region); - - touch(ready_path); - while (access(go_path, F_OK) != 0) { - usleep(1000); - } - - void* growth = mmap(NULL, region, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (growth == MAP_FAILED) { - return 1; - } - memset(growth, 0x42, region); - - int ret = write_rss_report(report_path); - munmap(baseline, region); - munmap(growth, region); - return ret; -} diff --git a/crates/memtrack/testdata/rss/rmap_leader_exit.c b/crates/memtrack/testdata/rss/rmap_leader_exit.c deleted file mode 100644 index 51e7b23c..00000000 --- a/crates/memtrack/testdata/rss/rmap_leader_exit.c +++ /dev/null @@ -1,74 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "rss_report.h" - -/* The main thread publishes its tid and leaves via pthread_exit while a worker - * thread keeps the process alive. The worker waits until the leader is a - * zombie — its exit path, including the sched_process_exit tracepoint, has - * fully run — then faults an anon region, writes the report, and exits the - * whole group. A tracker keyed on leader exit instead of thread-group death - * would stop watching before the worker's region is faulted. - * - * argv: [1]=report path. */ - -static const char* report_path; -static pid_t leader_tid; - -static int leader_is_zombie(void) { - char path[64]; - char buf[256]; - snprintf(path, sizeof(path), "/proc/self/task/%d/stat", leader_tid); - FILE* f = fopen(path, "r"); - if (!f) { - return 1; - } - size_t n = fread(buf, 1, sizeof(buf) - 1, f); - fclose(f); - buf[n] = '\0'; - /* The state field follows the parenthesized comm. */ - const char* p = strrchr(buf, ')'); - if (!p || p[1] == '\0' || p[2] == '\0') { - return 0; - } - return p[2] == 'Z'; -} - -static void* worker(void* arg) { - (void)arg; - while (!leader_is_zombie()) { - usleep(1000); - } - - size_t region = 64UL * 1024 * 1024; - void* mem = - mmap(NULL, region, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (mem == MAP_FAILED) { - _exit(1); - } - memset(mem, 0x42, region); - - /* The leader is a zombie by now, so /proc//status has no Rss lines; - * report through this worker's own task instead. */ - int ret = write_rss_report_pid((int)syscall(SYS_gettid), report_path); - munmap(mem, region); - _exit(ret); -} - -int main(int argc, char** argv) { - if (argc != 2) { - return 1; - } - report_path = argv[1]; - leader_tid = (pid_t)syscall(SYS_gettid); - - pthread_t thread; - if (pthread_create(&thread, NULL, worker, NULL) != 0) { - return 1; - } - pthread_exit(NULL); -} diff --git a/crates/memtrack/tests/rss_tests.rs b/crates/memtrack/tests/rss_tests.rs index 41a044a4..197f7037 100644 --- a/crates/memtrack/tests/rss_tests.rs +++ b/crates/memtrack/tests/rss_tests.rs @@ -15,6 +15,12 @@ fn mib_16(bytes: u64) -> u64 { (bytes + 8 * MIB) / (16 * MIB) * 16 } +fn page_size() -> u64 { + let size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + assert!(size > 0, "sysconf(_SC_PAGESIZE) failed"); + size as u64 +} + fn parse_report(report: &str) -> BTreeMap { report .lines() @@ -211,8 +217,8 @@ fn assert_rmap_hole_addresses( hole_len: u64, len: u64, ) { - const PAGE: u64 = 4096; - let n_pages = (len / PAGE) as usize; + let page = page_size(); + let n_pages = (len / page) as usize; let mut first_remove = vec![u64::MAX; n_pages]; let mut added = vec![false; n_pages]; let mut readded = vec![false; n_pages]; @@ -227,7 +233,7 @@ fn assert_rmap_hole_addresses( if event.addr < base || event.addr >= base + len { continue; } - let first = ((event.addr - base) / PAGE) as usize; + let first = ((event.addr - base) / page) as usize; let last = (first + delta.unsigned_abs() as usize).min(n_pages); for page in first..last { if delta > 0 { @@ -241,7 +247,7 @@ fn assert_rmap_hole_addresses( } } - let hole = (hole_off / PAGE) as usize..((hole_off + hole_len) / PAGE) as usize; + let hole = (hole_off / page) as usize..((hole_off + hole_len) / page) as usize; for page in 0..n_pages { assert!(added[page], "page {page} never saw an rmap add"); assert_eq!( @@ -326,308 +332,98 @@ fn test_rss_rmap_tracking( Ok(()) } -#[test_with::env(GITHUB_ACTIONS)] -#[test] -fn test_rss_external_reclaim() -> Result<(), Box> { - let temp_dir = TempDir::new()?; - let binary = shared::compile_c_source( - include_str!("../testdata/rss/madvise_extern.c"), - "madvise_extern", - temp_dir.path(), - )?; - let (events, handle) = shared::track_command(Command::new(&binary))?; - handle.join().unwrap(); - - // Single fork: parent_pid == A (owner), event.pid == B (external caller, single-threaded - // so its tid == its pid). - let (a, b) = events - .iter() - .find_map(|e| match e.kind { - MemtrackEventKind::Fork { parent_pid } => Some((parent_pid, e.pid)), - _ => None, - }) - .expect("expected a fork event"); - - let peak = events - .iter() - .filter_map(|e| match e.kind { - MemtrackEventKind::Rss { member: 0, size } if e.pid == a => Some(size), - _ => None, - }) - .max() - .unwrap_or(0); - assert!(peak >= 32 * MIB, "peak file RSS too small: {peak}"); - - // A file decrement owned by A but emitted from B's context: only present when - // out-of-context rss_stat updates are attributed to the owning process. - let external_decrement = events.iter().any(|e| { - e.pid == a - && e.tid == b - && matches!(e.kind, MemtrackEventKind::Rss { member: 0, size } if size < peak) - }); - assert!( - external_decrement, - "external file-RSS decrement not attributed to A (tid=B)" - ); - Ok(()) +/// A foreign actor reclaiming another process's memory must be attributed to +/// the OWNER of that memory, not the actor. The fixture forks a child B that +/// calls process_madvise(MADV_PAGEOUT) against parent A's file region from B's +/// own context (tid == B). Both accounting modes must show A's in-context faults +/// AND the foreign reclaim charged back to A via the mm_owner map. +enum Reclaim { + /// Absolute file-RSS updates; a foreign reclaim appears as a decrement. + RssStat, + /// Reconstructed file-page deltas; a foreign reclaim appears as removes. + Rmap, } #[test_with::env(GITHUB_ACTIONS)] -#[test] -fn test_rss_rmap_external_reclaim() -> Result<(), Box> { - let temp_dir = TempDir::new()?; - let binary = shared::compile_c_source( +#[rstest] +#[case::rss_stat(Reclaim::RssStat)] +#[case::rmap(Reclaim::Rmap)] +fn test_rss_external_reclaim(#[case] mode: Reclaim) -> Result<(), Box> { + let track: fn(Command) -> shared::TrackResult = match mode { + Reclaim::RssStat => shared::track_command, + Reclaim::Rmap => shared::track_command_with_rmap, + }; + let (_report, events) = track_fixture( include_str!("../testdata/rss/madvise_extern.c"), "madvise_extern", - temp_dir.path(), + track, )?; - let (events, handle) = shared::track_command_with_rmap(Command::new(&binary))?; - handle.join().unwrap(); - // Single fork: parent_pid == A (owner that faulted the file region), event.pid == B - // (external caller of process_madvise(MADV_PAGEOUT) against A from its own context). - let (a, b) = events - .iter() - .find_map(|e| match e.kind { - MemtrackEventKind::Fork { parent_pid } => Some((parent_pid, e.pid)), - _ => None, - }) - .expect("expected a fork event"); - - // Sanity: A's own in-context file-page faults are reconstructed by rmap. - let in_context_add: i64 = events - .iter() - .filter_map(|e| match e.kind { - MemtrackEventKind::Rmap { member: 0, delta } if e.pid == a && delta > 0 => Some(delta), - _ => None, - }) - .sum(); - let in_context_bytes = in_context_add as u64 * 4096; - assert!( - in_context_bytes >= 32 * MIB, - "in-context file rmap adds too small: {in_context_bytes}" - ); - - // The point of the test: a file-page remove owned by A but emitted from B's - // context (tid == B), only present when the foreign reclaim's rmap events are - // attributed to the owning process via the mm_owner map. - let external_removed: i64 = events - .iter() - .filter_map(|e| match e.kind { - MemtrackEventKind::Rmap { member: 0, delta } - if e.pid == a && e.tid == b && delta < 0 => - { - Some(-delta) - } - _ => None, - }) - .sum(); - assert!( - external_removed > 0, - "external MADV_PAGEOUT remove not attributed to the owner (pid=A, tid=B)" - ); - let external_bytes = external_removed as u64 * 4096; - assert!( - external_bytes >= 8 * MIB, - "external MADV_PAGEOUT remove not attributed to the owner: only {external_bytes} bytes" - ); - Ok(()) -} - -/// TEMPORARY diagnostic: track a real-world workload (`ls /nix/store`, ~50 MiB -/// peak on a populated store) and cross-check the reconstructed rss_stat and -/// rmap peaks against the kernel's own accounting (`wait4` ru_maxrss) from an -/// identical untracked run. `ls` is single-process, so raw byte peaks are -/// accumulated directly without per-pid splitting. -#[test_with::env(GITHUB_ACTIONS)] -#[test] -fn test_rss_ls_nix_store() -> Result<(), Box> { - if !std::path::Path::new("/nix/store").is_dir() { - eprintln!("skipping: /nix/store not available"); - return Ok(()); - } - - let ls_command = || { - let mut cmd = Command::new("ls"); - cmd.arg("/nix/store").stdout(std::process::Stdio::null()); - cmd - }; - - // Ground truth: identical untracked run, reaped via wait4 for ru_maxrss. - let child = ls_command().spawn()?; - let pid = child.id() as i32; - let mut status = 0i32; - let mut rusage: libc::rusage = unsafe { std::mem::zeroed() }; - let reaped = unsafe { libc::wait4(pid, &mut status, 0, &mut rusage) }; - assert_eq!(reaped, pid, "wait4 failed"); - assert!( - libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0, - "untracked ls failed: status {status}" - ); - let truth_bytes = rusage.ru_maxrss as u64 * 1024; - - let (events, handle) = shared::track_command_with_rmap(ls_command())?; - handle.join().unwrap(); + // A = owner that faulted the file region; B = external caller, single-threaded + // so its tid == its pid. + let (a, b) = first_fork_pair(&events).expect("expected a fork event"); + + match mode { + Reclaim::RssStat => { + let peak = events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Rss { member: 0, size } if e.pid == a => Some(size), + _ => None, + }) + .max() + .unwrap_or(0); + assert!(peak >= 32 * MIB, "peak file RSS too small: {peak}"); + + // A file decrement owned by A but emitted from B's context: only present + // when out-of-context rss_stat updates are attributed to the owner. + let external_decrement = events.iter().any(|e| { + e.pid == a + && e.tid == b + && matches!(e.kind, MemtrackEventKind::Rss { member: 0, size } if size < peak) + }); + assert!( + external_decrement, + "external file-RSS decrement not attributed to A (tid=B)" + ); + } + Reclaim::Rmap => { + let in_context_bytes = events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Rmap { member: 0, delta } if e.pid == a && delta > 0 => { + Some(delta) + } + _ => None, + }) + .sum::() as u64 + * page_size(); + assert!( + in_context_bytes >= 32 * MIB, + "in-context file rmap adds too small: {in_context_bytes}" + ); - let mut rss = RssAccum::default(); - let mut rmap = RmapAccum::default(); - for event in events.iter().sorted_by_key(|event| event.timestamp) { - match event.kind { - MemtrackEventKind::Rss { member, size } => { - if let Ok(index @ 0..4) = usize::try_from(member) { - rss.latest[index] = size; - rss.update_peaks(); - } - } - MemtrackEventKind::Rmap { member, delta } => { - if let Ok(index @ 0..4) = usize::try_from(member) { - rmap.totals[index] += delta * 4096; - rmap.update_peaks(); - } - } - _ => {} + // A file-page remove owned by A but emitted from B's context (tid == B), + // only present when the foreign reclaim's rmap events are attributed to + // the owner via the mm_owner map. + let external_bytes = events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Rmap { member: 0, delta } + if e.pid == a && e.tid == b && delta < 0 => + { + Some(-delta) + } + _ => None, + }) + .sum::() as u64 + * page_size(); + assert!( + external_bytes >= 8 * MIB, + "external MADV_PAGEOUT remove not attributed to the owner (pid=A, tid=B): only {external_bytes} bytes" + ); } } - - let rss_bytes = rss.max_rss; - let rmap_bytes = rmap.max_rss.max(0) as u64; - eprintln!( - "ls /nix/store max RSS: wait4={:.1} MiB rss_stat={:.1} MiB rmap={:.1} MiB", - truth_bytes as f64 / MIB as f64, - rss_bytes as f64 / MIB as f64, - rmap_bytes as f64 / MIB as f64, - ); - - let within = |measured: u64| { - (truth_bytes as f64 * 0.8..=truth_bytes as f64 * 1.2).contains(&(measured as f64)) - }; - assert!( - within(rss_bytes), - "rss_stat peak {rss_bytes} outside 20% of wait4 {truth_bytes}" - ); - assert!( - within(rmap_bytes), - "rmap peak {rmap_bytes} outside 20% of wait4 {truth_bytes}" - ); - Ok(()) -} - -/// rss_stat is an absolute kernel counter; the rmap estimate is reconstructed -/// from zero by summing folio add/remove deltas. Both are emitted for the -/// whole lifetime of a tracked pid, independent of the enable toggle (which -/// only gates allocator events): a delta stream that starts mid-life could -/// never recover the resident baseline faulted before enable. -/// -/// The fixture faults a 64 MiB anon baseline before `enable_tracking`, then a -/// 64 MiB anon region after. Reduced with the same Exec-reset lifecycle the -/// production parser uses, both series peak at ~128 MiB: the pre-enable -/// baseline is visible to rmap because the pid is tracked from spawn. -#[test_with::env(GITHUB_ACTIONS)] -#[test] -fn test_rss_rmap_late_enable_covers_baseline() -> Result<(), Box> { - const REGION_MIB: u64 = 64; - - let temp_dir = TempDir::new()?; - std::fs::write( - temp_dir.path().join("rss_report.h"), - include_str!("../testdata/rss/rss_report.h"), - )?; - let binary = shared::compile_c_source( - include_str!("../testdata/rss/rmap_late_enable.c"), - "rmap_late_enable", - temp_dir.path(), - )?; - let report_path = temp_dir.path().join("rmap_late_enable.report"); - let ready_path = temp_dir.path().join("ready"); - let go_path = temp_dir.path().join("go"); - - let mut command = Command::new(&binary); - command.arg(&report_path).arg(&ready_path).arg(&go_path); - - let (events, handle) = - shared::track_command_with_rmap_late_enable(command, &ready_path, &go_path)?; - handle.join().unwrap(); - - let (rss_stat, rmap) = per_pid_peaks(&events); - let rss = rss_stat.first().ok_or("no rss_stat pid observed")?; - let rmap = rmap.first().ok_or("no rmap pid observed")?; - eprintln!( - "late-enable anon peaks: rss_stat={} MiB rmap={} MiB (region={} MiB each)", - rss.anon_mib, rmap.anon_mib, REGION_MIB - ); - - // rss_stat's absolute counter covers baseline + growth. - assert!( - rss.anon_mib >= 2 * REGION_MIB - 16, - "rss_stat anon peak {} MiB below the expected ~{} MiB baseline+growth", - rss.anon_mib, - 2 * REGION_MIB - ); - // The rmap accumulator covers the pre-enable baseline too: lifetime - // events are gated on is_tracked, not is_enabled. - assert!( - rmap.anon_mib >= 2 * REGION_MIB - 16, - "rmap anon peak {} MiB misses the pre-enable baseline; expected ~{} MiB", - rmap.anon_mib, - 2 * REGION_MIB - ); - let gap = rss.anon_mib.abs_diff(rmap.anon_mib); - assert!( - gap <= 16, - "rss_stat ({} MiB) and rmap ({} MiB) diverged by {} MiB despite lifetime tracking", - rss.anon_mib, - rmap.anon_mib, - gap - ); - Ok(()) -} - -/// The leader thread can pthread_exit while worker threads keep the process -/// alive; EXIT must mark thread-group death, not leader exit. The fixture's -/// worker faults a 64 MiB anon region only after the leader is a zombie, so -/// an exit path keyed on the leader would untrack the pid before the region -/// is faulted and emit EXIT ahead of the worker's rmap events. -#[test_with::env(GITHUB_ACTIONS)] -#[test] -fn test_rss_rmap_leader_exit_keeps_tracking() -> Result<(), Box> { - const REGION_MIB: u64 = 64; - - let (_raw_report, events) = track_fixture( - include_str!("../testdata/rss/rmap_leader_exit.c"), - "rmap_leader_exit", - shared::track_command_with_rmap, - )?; - - // The fixture process is the pid with the largest anon rmap peak. - let (_rss_stat, rmap) = per_pid_peaks(&events); - let rmap_peak = rmap - .iter() - .max_by_key(|p| p.anon_mib) - .ok_or("no rmap pid observed")?; - assert!( - rmap_peak.anon_mib >= REGION_MIB - 16, - "rmap anon peak {} MiB misses the worker's post-leader-exit region (~{} MiB)", - rmap_peak.anon_mib, - REGION_MIB - ); - - let pid = rmap_peak.pid; - let exits: Vec<&MemtrackEvent> = events - .iter() - .filter(|e| e.pid == pid && matches!(e.kind, MemtrackEventKind::Exit)) - .collect(); - assert_eq!(exits.len(), 1, "expected exactly one EXIT for pid {pid}"); - - let last_rmap_ts = events - .iter() - .filter(|e| e.pid == pid && matches!(e.kind, MemtrackEventKind::Rmap { .. })) - .map(|e| e.timestamp) - .max() - .ok_or("no rmap events for fixture pid")?; - assert!( - exits[0].timestamp > last_rmap_ts, - "EXIT fired before the worker's rmap events: leader exit was treated as process death" - ); Ok(()) } @@ -647,13 +443,8 @@ fn test_rss_rmap_thread_fork_tracks_child() -> Result<(), Box Some((parent_pid, e.pid)), - _ => None, - }) - .ok_or("no fork event: worker-thread fork was not tracked")?; + let (parent, child) = + first_fork_pair(&events).ok_or("no fork event: worker-thread fork was not tracked")?; assert_ne!(parent, child); let child_anon: i64 = events @@ -666,9 +457,9 @@ fn test_rss_rmap_thread_fork_tracks_child() -> Result<(), Box= ((REGION_MIB - 16) * MIB) as i64, + child_anon * page_size() as i64 >= ((REGION_MIB - 16) * MIB) as i64, "child of a worker-thread fork missed rmap tracking: anon adds {} bytes, expected ~{} MiB", - child_anon * 4096, + child_anon * page_size() as i64, REGION_MIB ); Ok(()) diff --git a/crates/memtrack/tests/shared.rs b/crates/memtrack/tests/shared.rs index 9d5a040a..f38edb7c 100644 --- a/crates/memtrack/tests/shared.rs +++ b/crates/memtrack/tests/shared.rs @@ -214,50 +214,3 @@ fn track_command_impl(command: Command, track_rmap: bool) -> TrackResult { Ok((events, thread_handle)) } - -/// Track a command with rmap, enabling tracking only after the target creates -/// `ready_path`. The target is spawned and resumed first, so any memory it -/// faults before signalling `ready` is already resident when tracking turns on. -/// The caller enables tracking, then creates `go_path` to release the target. -pub fn track_command_with_rmap_late_enable( - command: Command, - ready_path: &Path, - go_path: &Path, -) -> TrackResult { - let tracker = Tracker::new_without_allocators_with_rmap(true)?; - - let mut session = tracker.spawn(&command, None)?; - let rx = session.take_events()?; - - let handshake = (|| -> anyhow::Result<()> { - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); - while !ready_path.exists() { - if std::time::Instant::now() > deadline { - anyhow::bail!("target never signalled baseline-ready"); - } - std::thread::sleep(std::time::Duration::from_millis(2)); - } - tracker.enable_tracking()?; - std::fs::write(go_path, b"go")?; - Ok(()) - })(); - - // A failed handshake leaves the target blocked on `go_path`; Session has no - // Drop kill, so reap it explicitly before propagating or the test hangs. - if let Err(e) = handshake { - unsafe { libc::kill(session.pid(), libc::SIGKILL) }; - let _ = session.wait(); - let _ = tracker.finish(); - return Err(e); - } - - session.wait()?; - drop(session); - let events: Vec = rx.iter().collect(); - - tracker.finish()?; - let thread_handle = std::thread::spawn(move || drop(tracker)); - - info!("Tracked {} events (late enable)", events.len()); - Ok((events, thread_handle)) -} From 2024266528b1344abf3d1b8a9464723eeba6154f Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 20 Jul 2026 18:07:42 +0200 Subject: [PATCH 08/13] ci(memtrack): run bpf tests on arm and add rss_tests shard Add an aarch64 lane to the bpf-tests matrix and run the rss integration tests alongside the existing test binaries. --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6935beb..e9f9e7d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,14 +88,15 @@ jobs: run: cargo run -- exec -m walltime --skip-upload --warmup-time 0s --max-rounds 5 -- ls -la bpf-tests: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: + os: [ubuntu-latest, ubuntu-24.04-arm] # Each memtrack integration test binary runs its cases serially # (eBPF tracker can't overlap with itself in one process), so we # shard at the test-binary level to parallelize across jobs. - test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests] + test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests, rss_tests] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -103,7 +104,7 @@ jobs: submodules: true - uses: ./.github/actions/install-rust with: - cache-key: ${{ matrix.test }} + cache-key: ${{ matrix.os }}-${{ matrix.test }} - uses: ./.github/actions/install-bpf-deps - name: Install additional allocators From 08b8575d5f648914a1592964f06bb0e11db7768f Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 24 Jul 2026 12:21:33 +0200 Subject: [PATCH 09/13] fixup! feat(memtrack): reconstruct RSS from gated folio rmap fentry hooks --- crates/memtrack/src/ebpf/c/event.h | 2 +- crates/memtrack/src/ebpf/memtrack/macros.rs | 26 +++--- crates/memtrack/src/ebpf/memtrack/mod.rs | 84 +++++-------------- crates/memtrack/src/ebpf/memtrack/tracking.rs | 82 +++++------------- crates/memtrack/src/ebpf/tracker.rs | 36 ++++---- 5 files changed, 74 insertions(+), 156 deletions(-) diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index 5dabffbe..50a60846 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -57,7 +57,7 @@ struct event { } rss; struct { - int32_t member; /* MM_* counter index, same values as rss.member */ + int32_t member; /* MM_* counter index */ int64_t delta; uint64_t addr; } rmap; diff --git a/crates/memtrack/src/ebpf/memtrack/macros.rs b/crates/memtrack/src/ebpf/memtrack/macros.rs index f4eca485..32c6082a 100644 --- a/crates/memtrack/src/ebpf/memtrack/macros.rs +++ b/crates/memtrack/src/ebpf/memtrack/macros.rs @@ -135,17 +135,19 @@ macro_rules! attach_tracepoint { }; } -macro_rules! attach_fentry { - ($func:ident, $prog:ident) => { - fn $func(&mut self) -> Result<()> { - let link = self - .skel - .progs - .$prog - .attach() - .context(format!("Failed to attach {} fentry", stringify!($prog)))?; - self.probes.push(link); - Ok(()) - } +/// Invokes `$cb!()` once per folio rmap fentry target. `` is the +/// kernel function base name; the BPF program is `fentry_`, so a callback +/// builds the program/method idents from `` with `paste!`. +macro_rules! for_each_rmap_prog { + ($cb:ident) => { + $cb!(folio_add_new_anon_rmap); + $cb!(folio_add_anon_rmap_ptes); + $cb!(folio_add_anon_rmap_pmd); + $cb!(folio_add_file_rmap_ptes); + $cb!(folio_add_file_rmap_pmd); + $cb!(folio_add_file_rmap_pud); + $cb!(folio_remove_rmap_ptes); + $cb!(folio_remove_rmap_pmd); + $cb!(folio_remove_rmap_pud); }; } diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index f40f4c1b..47a075ad 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -103,11 +103,6 @@ pub struct MemtrackBpf { } impl MemtrackBpf { - pub fn new() -> Result { - let track_rmap = std::env::var("CODSPEED_MEMTRACK_TRACK_RMAP").is_ok_and(|v| v == "1"); - Self::new_with_rmap(track_rmap) - } - pub fn new_with_rmap(track_rmap: bool) -> Result { let builder = MainSkelBuilder::default(); let open_object = Box::leak(Box::new(MaybeUninit::uninit())); @@ -124,72 +119,31 @@ impl MemtrackBpf { let mut btf_disabled_rmap_targets = Vec::new(); if !track_rmap { - open_skel - .progs - .fentry_folio_add_new_anon_rmap - .set_autoload(false); - open_skel - .progs - .fentry_folio_add_anon_rmap_ptes - .set_autoload(false); - open_skel - .progs - .fentry_folio_add_anon_rmap_pmd - .set_autoload(false); - open_skel - .progs - .fentry_folio_add_file_rmap_ptes - .set_autoload(false); - open_skel - .progs - .fentry_folio_add_file_rmap_pmd - .set_autoload(false); - open_skel - .progs - .fentry_folio_add_file_rmap_pud - .set_autoload(false); - open_skel - .progs - .fentry_folio_remove_rmap_ptes - .set_autoload(false); - open_skel - .progs - .fentry_folio_remove_rmap_pmd - .set_autoload(false); - open_skel - .progs - .fentry_folio_remove_rmap_pud - .set_autoload(false); + macro_rules! disable_rmap_prog { + ($name:ident) => { + paste::paste! { + open_skel.progs.[].set_autoload(false); + } + }; + } + for_each_rmap_prog!(disable_rmap_prog); } else { let btf = libbpf_rs::btf::Btf::from_vmlinux().context("Failed to load vmlinux BTF")?; macro_rules! disable_missing_kernel_func { - ($prog:ident, $target:literal) => { - if !kernel_func_exists(&btf, $target) { - open_skel.progs.$prog.set_autoload(false); - btf_disabled_rmap_targets.push($target); - warn!( - "Kernel function {} not present in BTF, disabling rmap fentry", - $target - ); + ($name:ident) => { + paste::paste! { + if !kernel_func_exists(&btf, stringify!($name)) { + open_skel.progs.[].set_autoload(false); + btf_disabled_rmap_targets.push(stringify!($name)); + warn!( + "Kernel function {} not present in BTF, disabling rmap fentry", + stringify!($name) + ); + } } }; } - - disable_missing_kernel_func!(fentry_folio_add_new_anon_rmap, "folio_add_new_anon_rmap"); - disable_missing_kernel_func!( - fentry_folio_add_anon_rmap_ptes, - "folio_add_anon_rmap_ptes" - ); - disable_missing_kernel_func!(fentry_folio_add_anon_rmap_pmd, "folio_add_anon_rmap_pmd"); - disable_missing_kernel_func!( - fentry_folio_add_file_rmap_ptes, - "folio_add_file_rmap_ptes" - ); - disable_missing_kernel_func!(fentry_folio_add_file_rmap_pmd, "folio_add_file_rmap_pmd"); - disable_missing_kernel_func!(fentry_folio_add_file_rmap_pud, "folio_add_file_rmap_pud"); - disable_missing_kernel_func!(fentry_folio_remove_rmap_ptes, "folio_remove_rmap_ptes"); - disable_missing_kernel_func!(fentry_folio_remove_rmap_pmd, "folio_remove_rmap_pmd"); - disable_missing_kernel_func!(fentry_folio_remove_rmap_pud, "folio_remove_rmap_pud"); + for_each_rmap_prog!(disable_missing_kernel_func); } let skel = Box::new( diff --git a/crates/memtrack/src/ebpf/memtrack/tracking.rs b/crates/memtrack/src/ebpf/memtrack/tracking.rs index 38e11a93..5f97fbd7 100644 --- a/crates/memtrack/src/ebpf/memtrack/tracking.rs +++ b/crates/memtrack/src/ebpf/memtrack/tracking.rs @@ -7,42 +7,6 @@ impl MemtrackBpf { attach_tracepoint!(task_newtask); attach_tracepoint!(sched_process_exec); attach_tracepoint!(sched_process_exit); - attach_fentry!( - attach_fentry_folio_add_new_anon_rmap, - fentry_folio_add_new_anon_rmap - ); - attach_fentry!( - attach_fentry_folio_add_anon_rmap_ptes, - fentry_folio_add_anon_rmap_ptes - ); - attach_fentry!( - attach_fentry_folio_add_anon_rmap_pmd, - fentry_folio_add_anon_rmap_pmd - ); - attach_fentry!( - attach_fentry_folio_add_file_rmap_ptes, - fentry_folio_add_file_rmap_ptes - ); - attach_fentry!( - attach_fentry_folio_add_file_rmap_pmd, - fentry_folio_add_file_rmap_pmd - ); - attach_fentry!( - attach_fentry_folio_add_file_rmap_pud, - fentry_folio_add_file_rmap_pud - ); - attach_fentry!( - attach_fentry_folio_remove_rmap_ptes, - fentry_folio_remove_rmap_ptes - ); - attach_fentry!( - attach_fentry_folio_remove_rmap_pmd, - fentry_folio_remove_rmap_pmd - ); - attach_fentry!( - attach_fentry_folio_remove_rmap_pud, - fentry_folio_remove_rmap_pud - ); fn rmap_target_enabled(&self, target: &str) -> bool { !self.btf_disabled_rmap_targets.contains(&target) @@ -56,33 +20,25 @@ impl MemtrackBpf { warn!("Failed to attach rss_stat tracepoint, RSS collection disabled: {e:#}"); } if self.track_rmap { - if self.rmap_target_enabled("folio_add_new_anon_rmap") { - self.attach_fentry_folio_add_new_anon_rmap()?; - } - if self.rmap_target_enabled("folio_add_anon_rmap_ptes") { - self.attach_fentry_folio_add_anon_rmap_ptes()?; - } - if self.rmap_target_enabled("folio_add_anon_rmap_pmd") { - self.attach_fentry_folio_add_anon_rmap_pmd()?; - } - if self.rmap_target_enabled("folio_add_file_rmap_ptes") { - self.attach_fentry_folio_add_file_rmap_ptes()?; - } - if self.rmap_target_enabled("folio_add_file_rmap_pmd") { - self.attach_fentry_folio_add_file_rmap_pmd()?; - } - if self.rmap_target_enabled("folio_add_file_rmap_pud") { - self.attach_fentry_folio_add_file_rmap_pud()?; - } - if self.rmap_target_enabled("folio_remove_rmap_ptes") { - self.attach_fentry_folio_remove_rmap_ptes()?; - } - if self.rmap_target_enabled("folio_remove_rmap_pmd") { - self.attach_fentry_folio_remove_rmap_pmd()?; - } - if self.rmap_target_enabled("folio_remove_rmap_pud") { - self.attach_fentry_folio_remove_rmap_pud()?; - } + macro_rules! attach_rmap_prog { + ($name:ident) => { + paste! { + if self.rmap_target_enabled(stringify!($name)) { + let link = self + .skel + .progs + .[] + .attach() + .context(format!( + "Failed to attach {} fentry", + stringify!([]) + ))?; + self.probes.push(link); + } + } + }; + } + for_each_rmap_prog!(attach_rmap_prog); } Ok(()) } diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index 83b34853..aec6c5f2 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -18,19 +18,8 @@ impl Tracker { /// Create a new tracker. The exec-mapping watcher discovers and attaches /// allocator probes as the tracked process tree maps executable files. pub fn new() -> Result { - Self::bump_memlock_rlimit()?; - - let mut bpf = MemtrackBpf::new()?; - bpf.attach_tracepoints()?; - bpf.attach_exec_watcher()?; - - let bpf = Arc::new(Mutex::new(bpf)); - let worker = AttachWorker::start(bpf.clone())?; - - Ok(Self { - bpf, - worker: Mutex::new(Some(worker)), - }) + let track_rmap = std::env::var("CODSPEED_MEMTRACK_TRACK_RMAP").is_ok_and(|v| v == "1"); + Self::build(track_rmap, true) } /// Track per-process RSS via the rss_stat tracepoint and folio rmap fentry @@ -38,14 +27,31 @@ impl Tracker { /// runs, so `spawn` arms no on-demand allocator attachment — the tracker /// observes only tracepoints and (when `track_rmap`) the rmap fentries. pub fn new_without_allocators_with_rmap(track_rmap: bool) -> Result { + Self::build(track_rmap, false) + } + + /// Build a tracker: attach lifetime tracepoints (and rmap fentries when + /// `track_rmap`), plus, when `allocators` is set, the exec-mapping watcher + /// and the on-demand allocator-attach worker. + fn build(track_rmap: bool, allocators: bool) -> Result { Self::bump_memlock_rlimit()?; let mut bpf = MemtrackBpf::new_with_rmap(track_rmap)?; bpf.attach_tracepoints()?; + if allocators { + bpf.attach_exec_watcher()?; + } + + let bpf = Arc::new(Mutex::new(bpf)); + let worker = if allocators { + Some(AttachWorker::start(bpf.clone())?) + } else { + None + }; Ok(Self { - bpf: Arc::new(Mutex::new(bpf)), - worker: Mutex::new(None), + bpf, + worker: Mutex::new(worker), }) } From 1ca854664aa53f2836302d13353f0433ddd55da2 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 24 Jul 2026 12:22:44 +0200 Subject: [PATCH 10/13] fixup! feat(memtrack): emit fork/exec/exit lifecycle events --- crates/memtrack/src/ebpf/events.rs | 41 ++++++++---------------------- 1 file changed, 11 insertions(+), 30 deletions(-) diff --git a/crates/memtrack/src/ebpf/events.rs b/crates/memtrack/src/ebpf/events.rs index ccab7f5f..40e8a6c5 100644 --- a/crates/memtrack/src/ebpf/events.rs +++ b/crates/memtrack/src/ebpf/events.rs @@ -140,6 +140,12 @@ impl AttachRequest { mod tests { use super::*; + fn event_bytes(event: &bindings::event) -> &[u8] { + unsafe { + std::slice::from_raw_parts(event as *const _ as *const u8, std::mem::size_of_val(event)) + } + } + #[test] fn test_parse_realloc_event() { // Create a mock event with realloc data @@ -152,12 +158,7 @@ mod tests { event.data.realloc.new_addr = 0x2000; event.data.realloc.size = 256; - let bytes = unsafe { - std::slice::from_raw_parts( - &event as *const _ as *const u8, - std::mem::size_of_val(&event), - ) - }; + let bytes = event_bytes(&event); // Parse and validate: let parsed = parse_event(bytes).unwrap(); @@ -186,12 +187,7 @@ mod tests { event.data.alloc.addr = 0x1000; event.data.alloc.size = 128; - let bytes = unsafe { - std::slice::from_raw_parts( - &event as *const _ as *const u8, - std::mem::size_of_val(&event), - ) - }; + let bytes = event_bytes(&event); // Parse and validate: let parsed = parse_event(bytes).unwrap(); @@ -218,12 +214,7 @@ mod tests { event.data.rss.member = 1; event.data.rss.size = 4096 * 10; - let bytes = unsafe { - std::slice::from_raw_parts( - &event as *const _ as *const u8, - std::mem::size_of_val(&event), - ) - }; + let bytes = event_bytes(&event); let parsed = parse_event(bytes).unwrap(); assert_eq!(parsed.pid, 1000); @@ -249,12 +240,7 @@ mod tests { event.data.rmap.delta = 8; event.data.rmap.addr = 0x7f00; - let bytes = unsafe { - std::slice::from_raw_parts( - &event as *const _ as *const u8, - std::mem::size_of_val(&event), - ) - }; + let bytes = event_bytes(&event); let parsed = parse_event(bytes).unwrap(); assert_eq!(parsed.pid, 1000); @@ -278,12 +264,7 @@ mod tests { event.header.tid = 2000; event.data.fork.parent_pid = 1000; - let bytes = unsafe { - std::slice::from_raw_parts( - &event as *const _ as *const u8, - std::mem::size_of_val(&event), - ) - }; + let bytes = event_bytes(&event); let parsed = parse_event(bytes).unwrap(); assert_eq!(parsed.pid, 1001); From 355d0bb0c9f413626706cfabe95b218d70732a86 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 24 Jul 2026 12:22:45 +0200 Subject: [PATCH 11/13] fixup! test(memtrack): validate RSS and rmap reconstruction against /proc --- crates/memtrack/testdata/rss/file.c | 1 - crates/memtrack/testdata/rss/madvise_extern.c | 1 - crates/memtrack/tests/c_tests.rs | 29 +------------------ crates/memtrack/tests/shared.rs | 3 ++ 4 files changed, 4 insertions(+), 30 deletions(-) diff --git a/crates/memtrack/testdata/rss/file.c b/crates/memtrack/testdata/rss/file.c index 5d45c293..5c8ceff0 100644 --- a/crates/memtrack/testdata/rss/file.c +++ b/crates/memtrack/testdata/rss/file.c @@ -1,4 +1,3 @@ -#include #include #include #include diff --git a/crates/memtrack/testdata/rss/madvise_extern.c b/crates/memtrack/testdata/rss/madvise_extern.c index 4be4c6b2..388dbb33 100644 --- a/crates/memtrack/testdata/rss/madvise_extern.c +++ b/crates/memtrack/testdata/rss/madvise_extern.c @@ -1,6 +1,5 @@ #define _GNU_SOURCE #include -#include #include #include #include diff --git a/crates/memtrack/tests/c_tests.rs b/crates/memtrack/tests/c_tests.rs index 736e1a6a..2e29ae16 100644 --- a/crates/memtrack/tests/c_tests.rs +++ b/crates/memtrack/tests/c_tests.rs @@ -2,35 +2,8 @@ mod shared; use rstest::rstest; -use std::fs; -use std::path::Path; -use std::process::Command; use tempfile::TempDir; -/// Compiles C source code and returns the binary path -fn compile_c_source( - source_code: &str, - name: &str, - output_dir: &Path, -) -> Result> { - let source_path = output_dir.join(format!("{name}.c")); - let binary_path = output_dir.join(name); - - fs::write(&source_path, source_code)?; - - let output = Command::new("gcc") - .args(["-o", binary_path.to_str().unwrap()]) - .arg(&source_path) - .output()?; - - if !output.status.success() { - eprintln!("gcc stderr: {}", String::from_utf8_lossy(&output.stderr)); - return Err("Failed to compile C fixture".into()); - } - - Ok(binary_path) -} - struct AllocationTestCase { name: &'static str, source: &'static str, @@ -86,7 +59,7 @@ fn test_allocation_tracking( #[case] test_case: &AllocationTestCase, ) -> Result<(), Box> { let temp_dir = TempDir::new()?; - let binary = compile_c_source(test_case.source, test_case.name, temp_dir.path())?; + let binary = shared::compile_c_source(test_case.source, test_case.name, temp_dir.path())?; let (events, thread_handle) = shared::track_binary(&binary)?; diff --git a/crates/memtrack/tests/shared.rs b/crates/memtrack/tests/shared.rs index f38edb7c..5d9982ff 100644 --- a/crates/memtrack/tests/shared.rs +++ b/crates/memtrack/tests/shared.rs @@ -87,6 +87,9 @@ macro_rules! assert_events_with_marker { // Remove events outside our 0xC0D59EED marker allocations let filtered_events = $events .iter() + // Drop Rss before slicing: the marker window's skip(2) is positional + // (drops [marker-malloc, marker-free]), so a stray rss_stat event + // sorting between the pair would displace it and leak the marker free. .filter(|e| !matches!(e.kind, MemtrackEventKind::Rss { .. })) .sorted_by_key(|e| e.timestamp) .dedup_by(|a, b| a.addr == b.addr && discriminant(&a.kind) == discriminant(&b.kind)) From d25e3a044e14673204a2aef027237e40891343e2 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 24 Jul 2026 12:36:06 +0200 Subject: [PATCH 12/13] fixup! feat(memtrack): attribute foreign-actor rmap events via mm ownership --- crates/memtrack/src/ebpf/c/rss.bpf.h | 43 ++++++++----------- .../memtrack/src/ebpf/c/utils/event_helpers.h | 16 +++---- 2 files changed, 24 insertions(+), 35 deletions(-) diff --git a/crates/memtrack/src/ebpf/c/rss.bpf.h b/crates/memtrack/src/ebpf/c/rss.bpf.h index 2850a634..a1adcf96 100644 --- a/crates/memtrack/src/ebpf/c/rss.bpf.h +++ b/crates/memtrack/src/ebpf/c/rss.bpf.h @@ -168,8 +168,8 @@ static __always_inline int submit_rmap(struct vm_area_struct* vma, __s32 member, __u64 addr) { __u64 mm = (__u64)BPF_CORE_READ(vma, vm_mm); struct task_struct* task = bpf_get_current_task_btf(); - __u64 tid = bpf_get_current_pid_tgid(); - __u32 pid = tid >> 32; + __u32 pid = bpf_get_current_pid_tgid() >> 32; + __u32 owner; if ((__u64)BPF_CORE_READ(task, mm) == mm) { if (!is_tracked(pid)) { @@ -181,36 +181,31 @@ static __always_inline int submit_rmap(struct vm_area_struct* vma, __s32 member, * the lookups keep the hot path cheap AND keep both entries LRU-fresh, * since pid_mm is otherwise never read until exec/exit and could be * evicted independently of its still-hot mm_owner twin. */ - __u32* owner = bpf_map_lookup_elem(&mm_owner, &mm); - if (!owner || *owner != pid) { + __u32* reg = bpf_map_lookup_elem(&mm_owner, &mm); + if (!reg || *reg != pid) { bpf_map_update_elem(&mm_owner, &mm, &pid, BPF_ANY); } __u64* cur_mm = bpf_map_lookup_elem(&pid_mm, &pid); if (!cur_mm || *cur_mm != mm) { bpf_map_update_elem(&pid_mm, &pid, &mm, BPF_ANY); } - - SUBMIT_EVENT_AS(pid, EVENT_TYPE_RMAP, { - e->data.rmap.member = member; - e->data.rmap.delta = delta; - e->data.rmap.addr = addr; - }); - } - - /* Foreign actor (task->mm != mm, including kthreads whose task->mm is NULL): - * recover the owner from the in-context registration. Fail toward dropping - * the event on any uncertainty about ownership. */ - __u32* found = bpf_map_lookup_elem(&mm_owner, &mm); - if (!found) { - return 0; - } - __u32 owner = *found; - if (!is_tracked(owner)) { - return 0; + owner = pid; + } else { + /* Foreign actor (task->mm != mm, including kthreads whose task->mm is NULL): + * recover the owner from the in-context registration. Fail toward dropping + * the event on any uncertainty about ownership. */ + __u32* found = bpf_map_lookup_elem(&mm_owner, &mm); + if (!found) { + return 0; + } + owner = *found; + if (!is_tracked(owner)) { + return 0; + } } - /* SUBMIT_EVENT_AS stamps header.tid from the current task, identifying the - * foreign actor that performed the rmap change. */ + /* header.tid is stamped from the current task; for a foreign actor it + * identifies the performer, not the owning pid. */ SUBMIT_EVENT_AS(owner, EVENT_TYPE_RMAP, { e->data.rmap.member = member; e->data.rmap.delta = delta; diff --git a/crates/memtrack/src/ebpf/c/utils/event_helpers.h b/crates/memtrack/src/ebpf/c/utils/event_helpers.h index fbbc0536..fcaf5382 100644 --- a/crates/memtrack/src/ebpf/c/utils/event_helpers.h +++ b/crates/memtrack/src/ebpf/c/utils/event_helpers.h @@ -72,11 +72,14 @@ static __always_inline __u64* take_param(void* map) { return 0; \ } -#define SUBMIT_EVENT(evt_type, fill_data) \ +#define SUBMIT_GATED_EVENT(evt_type, fill_data) \ { \ + if (!is_enabled()) { \ + return 0; \ + } \ + \ __u64 tid = bpf_get_current_pid_tgid(); \ __u32 pid = tid >> 32; \ - \ if (!is_tracked(pid)) { \ return 0; \ } \ @@ -84,15 +87,6 @@ static __always_inline __u64* take_param(void* map) { SUBMIT_EVENT_AS(pid, evt_type, fill_data); \ } -#define SUBMIT_GATED_EVENT(evt_type, fill_data) \ - { \ - if (!is_enabled()) { \ - return 0; \ - } \ - \ - SUBMIT_EVENT(evt_type, fill_data); \ - } - static __always_inline int submit_alloc_event(__u64 size, __u64 addr) { SUBMIT_GATED_EVENT(EVENT_TYPE_MALLOC, { e->data.alloc.addr = addr; From 8a000e02b0ae7ce4af26f2ef3422a8e0c49fd3ba Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 24 Jul 2026 12:36:06 +0200 Subject: [PATCH 13/13] fixup! feat(memtrack): reconstruct RSS from gated folio rmap fentry hooks --- crates/memtrack/src/ebpf/c/rss.bpf.h | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/crates/memtrack/src/ebpf/c/rss.bpf.h b/crates/memtrack/src/ebpf/c/rss.bpf.h index a1adcf96..06e5b7e2 100644 --- a/crates/memtrack/src/ebpf/c/rss.bpf.h +++ b/crates/memtrack/src/ebpf/c/rss.bpf.h @@ -231,46 +231,45 @@ int BPF_PROG(fentry_folio_add_anon_rmap_pmd, struct folio* folio, struct page* p return submit_rmap(vma, MM_ANONPAGES, (__s64)folio_nr_pages_est(folio), address); } +static __always_inline int submit_file_rmap(struct folio* folio, struct page* page, + struct vm_area_struct* vma, __s64 delta) { + return submit_rmap(vma, folio_mm_counter(folio), delta, folio_page_address(folio, page, vma)); +} + SEC("fentry/folio_add_file_rmap_ptes") int BPF_PROG(fentry_folio_add_file_rmap_ptes, struct folio* folio, struct page* page, int nr_pages, struct vm_area_struct* vma) { - return submit_rmap(vma, folio_mm_counter(folio), (__s64)nr_pages, - folio_page_address(folio, page, vma)); + return submit_file_rmap(folio, page, vma, (__s64)nr_pages); } SEC("fentry/folio_add_file_rmap_pmd") int BPF_PROG(fentry_folio_add_file_rmap_pmd, struct folio* folio, struct page* page, struct vm_area_struct* vma) { - return submit_rmap(vma, folio_mm_counter(folio), (__s64)folio_nr_pages_est(folio), - folio_page_address(folio, page, vma)); + return submit_file_rmap(folio, page, vma, (__s64)folio_nr_pages_est(folio)); } SEC("fentry/folio_add_file_rmap_pud") int BPF_PROG(fentry_folio_add_file_rmap_pud, struct folio* folio, struct page* page, struct vm_area_struct* vma) { - return submit_rmap(vma, folio_mm_counter(folio), (__s64)folio_nr_pages_est(folio), - folio_page_address(folio, page, vma)); + return submit_file_rmap(folio, page, vma, (__s64)folio_nr_pages_est(folio)); } SEC("fentry/folio_remove_rmap_ptes") int BPF_PROG(fentry_folio_remove_rmap_ptes, struct folio* folio, struct page* page, int nr_pages, struct vm_area_struct* vma) { - return submit_rmap(vma, folio_mm_counter(folio), -(__s64)nr_pages, - folio_page_address(folio, page, vma)); + return submit_file_rmap(folio, page, vma, -(__s64)nr_pages); } SEC("fentry/folio_remove_rmap_pmd") int BPF_PROG(fentry_folio_remove_rmap_pmd, struct folio* folio, struct page* page, struct vm_area_struct* vma) { - return submit_rmap(vma, folio_mm_counter(folio), -(__s64)folio_nr_pages_est(folio), - folio_page_address(folio, page, vma)); + return submit_file_rmap(folio, page, vma, -(__s64)folio_nr_pages_est(folio)); } SEC("fentry/folio_remove_rmap_pud") int BPF_PROG(fentry_folio_remove_rmap_pud, struct folio* folio, struct page* page, struct vm_area_struct* vma) { - return submit_rmap(vma, folio_mm_counter(folio), -(__s64)folio_nr_pages_est(folio), - folio_page_address(folio, page, vma)); + return submit_file_rmap(folio, page, vma, -(__s64)folio_nr_pages_est(folio)); } /* == Process lifecycle events ==