diff --git a/.gitignore b/.gitignore index 5de7700d..d12dc425 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,6 @@ yarn-error.log* # Task runtime files .task/ + +# Python bytecode caches (scripts/skill-scan tooling) +__pycache__/ diff --git a/scripts/skill-scan/eval/.gitignore b/scripts/skill-scan/eval/.gitignore new file mode 100644 index 00000000..9ba5e67d --- /dev/null +++ b/scripts/skill-scan/eval/.gitignore @@ -0,0 +1,3 @@ +# Raw scan outputs, downloaded CI artifacts, and cloned skill sources — +# regenerable and large. Summary tables belong in the tracking issue. +results/ diff --git a/scripts/skill-scan/eval/README.md b/scripts/skill-scan/eval/README.md new file mode 100644 index 00000000..af8f826f --- /dev/null +++ b/scripts/skill-scan/eval/README.md @@ -0,0 +1,127 @@ +# skill-scanner model evaluation + +Benchmarks candidate LLM models for the skill-security-scan pipeline +(context: issue #904, PR #855). The current production model is set via the +`SKILL_SCANNER_LLM_MODEL` repo variable (`anthropic/claude-sonnet-4-6`). + +## Why + +Two problems with the current setup: + +1. **Nondeterminism.** The LLM analyzer re-flags already-reviewed content + under rotating rule IDs, so allowlists are a moving target (#904). +2. **Latency.** Some skills take 18+ minutes to scan (claude-api burned + ~333k input / ~69k output tokens in one scan). Wall time is dominated by + serial LLM output generation, so a faster model directly cuts scan time. + +Hypothesis: a faster/cheaper model produces similar blocking decisions on +this corpus. The current allowlists capture previously accepted findings, +but a new unallowlisted finding is not automatically noise: finding-level +review is required to distinguish scanner churn from a legitimate trust +boundary. Detection recall is measured separately on malicious fixtures. + +## Leg 1: Dockyard corpus (noise / stability / latency) + +```bash +python3 scripts/skill-scan/eval/bench_models.py --runs 3 +``` + +API keys come from a `.env` in this directory (gitignored) with +`ANTHROPIC_API_KEY` and `OPENAI_API_KEY`, or from per-provider key files: +`--key-file anthropic=~/keys/anthropic openai=~/keys/openai`. + +The default model matrix is the production baseline plus four candidates +(exact identifiers verified against the Anthropic and OpenAI model docs; +LiteLLM strings are provider-prefixed): + +| LiteLLM model string | Role | +|---|---| +| `anthropic/claude-sonnet-4-6` | production baseline | +| `anthropic/claude-sonnet-5` | Anthropic, sonnet-tier candidate | +| `anthropic/claude-haiku-4-5` | Anthropic, small/fast candidate | +| `openai/gpt-5.6-terra` | OpenAI, sonnet-equivalent candidate | +| `openai/gpt-5.6-luna` | OpenAI, small/fast candidate | + +Note the OpenAI IDs use a period in the version and a dash before the tier +(`gpt-5.6-terra`, not `gpt-5-6-terra`). Override with `--models` to run a +subset. + +On prefixes: the scanner passes the model string to LiteLLM unchanged +(`llm_provider_config.py`), so OpenAI models work bare (LiteLLM's default +provider) or with the explicit `openai/` prefix. We use the prefix for +clarity; the scanner's docs showing bare `gpt-*` names are just LiteLLM's +default-provider shorthand. Two scanner behaviors the harness compensates +for: `SKILL_SCANNER_LLM_API_KEY` is the key for *any* provider (there is no +per-provider key var, hence `--key-file provider=path`), and the scanner +sends `temperature` unconditionally, which GPT-5.x models reject, so the +harness sets `SKILL_SCANNER_LLM_TEMPERATURE=none` (the scanner's +omit-the-parameter sentinel) for gpt-5* models. + +The default corpus is the nine skills with the worst churn/latency history +(claude-api, find-bugs, pulumi-upgrade-provider, provider-upgrade, +huggingface-paper-publisher, codeql, semgrep, supply-chain-risk-auditor, +zeroize-audit). Each skill is scanned at the exact `spec.ref` pinned in its +spec.yaml, with `SKILL_SCANNER_LLM_TEMPERATURE=0.0`, using the same +`run_scan.py` flags as CI. + +Metrics per skill x model (see `summary.md` in the results dir): + +- **Blocking/run**: findings not covered by the current allowlist. This is + the "PR fails CI again" event and measures operational churn. Inspect the + findings before calling them false positives. +- **HIGH+ noise/run**: blocking findings with the allowlist disabled, i.e. + total triage burden a maintainer would face from scratch. +- **LLM stability**: mean pairwise Jaccard similarity of the LLM-analyzer + finding sets across repetitions. 1.0 = deterministic. +- **Mean time / output tokens**: scan latency and its main driver. + +## Leg 2: detection recall (scanner's own ground-truth corpus) + +Dockyard has no known-malicious skills, so recall must come from the +scanner's eval framework, which ships curated malicious/safe skills with +`_expected.json` ground truth: + +```bash +git clone --branch 2.0.13 --depth 1 \ + https://github.com/cisco-ai-defense/skill-scanner \ + scripts/skill-scan/eval/results/recall-source-2.0.13 + +python3 scripts/skill-scan/eval/bench_recall.py \ + --scanner-source scripts/skill-scan/eval/results/recall-source-2.0.13 \ + --models anthropic/claude-sonnet-4-6 openai/gpt-5.6-terra \ + --runs 3 +``` + +Do not use the scanner's bundled `benchmark_runner.py` for a model +comparison: in scanner 2.0.13 it constructs `SkillScanner()` with the core +analyzers only and does not enable the LLM or meta analyzers. `bench_recall.py` +instead sends every fixture through Dockyard's production `run_scan.py` path. + +The recall summary reports fixture-level blocking decisions, safe-fixture +false positives, expected-category coverage at any severity and at HIGH+, +wall time, and token usage. A candidate must preserve fixture-level malicious +and safe decisions; severity/category differences should then be reviewed. + +## Decision criteria + +Prefer the cheapest/fastest model that, relative to `claude-sonnet-4-6`: + +- preserves malicious/safe fixture-level decisions on the recall corpus, +- has acceptable LLM stability and operational blocking churn, +- does not introduce materially worse finding-level false positives, +- and cuts p95 scan latency meaningfully. + +Aggregate counts are screening metrics, not a substitute for reviewing the +actual findings. A candidate can report more blockers because it catches a +real execution boundary, or fewer blockers because it misses one. + +## Evaluation records + +- [`reports/2026-08-26-sonnet-4.6-vs-terra.md`](reports/2026-08-26-sonnet-4.6-vs-terra.md) + records the initial Sonnet 4.6 versus GPT-5.6 Terra shootout. + +Orthogonal knobs worth testing in the same harness (both from #904): + +- `--consensus-runs 3` (majority vote inside the LLM analyzer; ~3x cost, + may let a cheap model match sonnet's stability at lower total cost) +- temperature pinning (already defaulted to 0.0 here) diff --git a/scripts/skill-scan/eval/bench_models.py b/scripts/skill-scan/eval/bench_models.py new file mode 100644 index 00000000..04fea799 --- /dev/null +++ b/scripts/skill-scan/eval/bench_models.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 +"""Benchmark skill-scanner LLM models against Dockyard's historical corpus. + +Runs the same scan pipeline CI uses (run_scan.py) against a set of skills, +once per model per repetition, and scores each model on: + + - latency: wall-clock scan duration + - blocking churn: findings that would block CI *today* (i.e. not covered + by the skill's current allowlist). This measures operational churn, but + findings still need review: some are legitimate trust boundaries rather + than scanner noise. + - noise volume: HIGH+ findings ignoring the allowlist entirely. This is + what a maintainer would have had to triage from scratch. Lower is better. + - stability: mean pairwise Jaccard similarity of the LLM-analyzer finding + sets across repetitions (1.0 = perfectly deterministic). Higher is better. + +Recall (does a cheaper model still catch real malware?) cannot be measured +from Dockyard data because the corpus contains no known-malicious skills. +Use the scanner's own ground-truth corpus for that leg — see README.md in +this directory. + +Usage: + python3 scripts/skill-scan/eval/bench_models.py \ + --key-file anthropic=~/keys/anthropic openai=~/keys/openai \ + --runs 3 + +The default --models matrix is the production baseline (sonnet-4-6) plus +claude-sonnet-5, claude-haiku-4-5, gpt-5.6-terra, and gpt-5.6-luna. + +Results land in scripts/skill-scan/eval/results// as raw scan +JSON plus summary.json and summary.md. +""" + +import argparse +import datetime +import itertools +import json +import os +import statistics +import subprocess +import sys +import time +from pathlib import Path + +import yaml + +EVAL_DIR = Path(__file__).resolve().parent +SKILL_SCAN_DIR = EVAL_DIR.parent +REPO_ROOT = SKILL_SCAN_DIR.parent.parent + +sys.path.insert(0, str(SKILL_SCAN_DIR)) +from process_scan_results import classify_findings, load_security_config # noqa: E402 + +# Skills with a history of scan churn or long scan times: +# - claude-api: 18+ min scans (see run 32855694687) +# - find-bugs, pulumi-upgrade-provider, provider-upgrade, +# huggingface-paper-publisher: re-flagged under rotating rule IDs (#904) +# - codeql, semgrep, supply-chain-risk-auditor, zeroize-audit: multiple +# allowlist rounds on PR #855 +DEFAULT_SKILLS = [ + "claude-api", + "find-bugs", + "pulumi-upgrade-provider", + "provider-upgrade", + "huggingface-paper-publisher", + "codeql", + "semgrep", + "supply-chain-risk-auditor", + "zeroize-audit", +] + + +def read_spec(skill: str) -> dict: + spec_file = REPO_ROOT / "skills" / skill / "spec.yaml" + with open(spec_file) as f: + spec = yaml.safe_load(f) + return { + "spec_file": str(spec_file), + "repository": spec["spec"]["repository"], + "ref": spec["spec"]["ref"], + "path": spec["spec"].get("path") or "", + } + + +def checkout_source(skill: str, meta: dict, cache_dir: Path) -> Path: + repo_dir = cache_dir / skill + if not (repo_dir / ".git").exists(): + repo_dir.parent.mkdir(parents=True, exist_ok=True) + subprocess.run( + ["git", "clone", "--filter=tree:0", "--no-checkout", "--quiet", + meta["repository"], str(repo_dir)], + check=True, + ) + subprocess.run( + ["git", "-C", str(repo_dir), "checkout", "--quiet", meta["ref"]], + check=True, + ) + src = repo_dir / meta["path"] if meta["path"] else repo_dir + if not src.is_dir(): + raise FileNotFoundError(f"skill source not found: {src}") + return src + + +def run_scan(source: Path, output: Path, model: str, api_key: str, + temperature: str, consensus_runs: int | None) -> tuple[float, bool]: + env = os.environ.copy() + env.update({ + "SKILL_SCANNER_USE_LLM": "true", + "SKILL_SCANNER_LLM_API_KEY": api_key, + "SKILL_SCANNER_LLM_MODEL": model, + "SKILL_SCANNER_LLM_TEMPERATURE": temperature, + }) + # GPT-5.x reasoning models reject non-default temperature; the scanner's + # "none" sentinel omits the parameter entirely (llm_request_handler.py, + # _TEMPERATURE_OMIT_VALUES). The meta-analyzer falls back to this same + # env var, so one setting covers both analyzers. + if "gpt-5" in model.lower(): + env["SKILL_SCANNER_LLM_TEMPERATURE"] = "none" + if consensus_runs and consensus_runs > 1: + env["SKILL_SCANNER_LLM_CONSENSUS_RUNS"] = str(consensus_runs) + else: + env.pop("SKILL_SCANNER_LLM_CONSENSUS_RUNS", None) + + start = time.monotonic() + proc = subprocess.run( + [sys.executable, str(SKILL_SCAN_DIR / "run_scan.py"), + "--source", str(source), "--output", str(output)], + env=env, capture_output=True, text=True, + ) + duration = time.monotonic() - start + if proc.returncode != 0 or not output.exists(): + sys.stderr.write(proc.stderr[-2000:] + "\n") + return duration, False + return duration, True + + +def finding_keys(scan: dict, analyzers: set[str] | None = None) -> set[tuple]: + keys = set() + for f in scan.get("findings") or []: + if not isinstance(f, dict): + continue + if analyzers and (f.get("analyzer") or "") not in analyzers: + continue + keys.add((f.get("analyzer"), f.get("rule_id"), f.get("file_path"))) + return keys + + +def sort_keys(keys: set[tuple]) -> list[tuple]: + # Findings can carry None fields (e.g. no file_path on skill-level LLM + # findings); plain sorted() dies comparing None with str. + return sorted(keys, key=lambda t: tuple("" if x is None else str(x) for x in t)) + + +def jaccard(a: set, b: set) -> float: + if not a and not b: + return 1.0 + return len(a & b) / len(a | b) + + +def summarize_cell(runs: list[dict]) -> dict: + ok = [r for r in runs if r["ok"]] + llm_sets = [set(map(tuple, r["llm_keys"])) for r in ok] + pair_sims = [jaccard(a, b) for a, b in itertools.combinations(llm_sets, 2)] + out_toks = [r["llm_output_tokens"] for r in ok if r.get("llm_output_tokens")] + return { + "llm_output_tokens_mean": round(statistics.mean(out_toks)) if out_toks else None, + "runs_ok": len(ok), + "runs_total": len(runs), + "duration_mean_s": round(statistics.mean(r["duration"] for r in runs), 1), + "duration_max_s": round(max(r["duration"] for r in runs), 1), + "blocking_per_run": [r["blocking"] for r in ok], + "noise_high_per_run": [r["noise_high"] for r in ok], + "llm_findings_per_run": [len(s) for s in llm_sets], + "llm_stability_jaccard": round(statistics.mean(pair_sims), 3) if pair_sims else None, + "llm_findings_union": [list(k) for k in sort_keys(set().union(*llm_sets))] if llm_sets else [], + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--skills", nargs="+", default=DEFAULT_SKILLS) + parser.add_argument("--models", nargs="+", default=[ + "anthropic/claude-sonnet-4-6", # production baseline + "anthropic/claude-sonnet-5", + "anthropic/claude-haiku-4-5", + "openai/gpt-5.6-terra", + "openai/gpt-5.6-luna", + ], help="LiteLLM model strings, e.g. anthropic/claude-haiku-4-5") + parser.add_argument("--runs", type=int, default=3) + parser.add_argument("--key-file", nargs="+", default=[], + help="Key file(s): either a single path used for all " + "models, or provider=path pairs, e.g. " + "anthropic=~/keys/ant openai=~/keys/oai. Falls back " + "to --env-file, then SKILL_SCANNER_LLM_API_KEY.") + parser.add_argument("--env-file", default=str(EVAL_DIR / ".env"), + help="dotenv file with ANTHROPIC_API_KEY / " + "OPENAI_API_KEY (default: .env in this directory)") + parser.add_argument("--temperature", default="0.0") + parser.add_argument("--consensus-runs", type=int, default=None, + help="Optional --llm-consensus-runs N passthrough") + parser.add_argument("--out", default=str(EVAL_DIR / "results")) + parser.add_argument("--resume", default=None, + help="Existing results dir (a previous run's timestamp " + "dir): scans already on disk are scored, not re-run") + args = parser.parse_args() + + keys: dict[str, str] = {} # provider prefix -> key ("" = default for all) + env_file = Path(args.env_file).expanduser() + if env_file.is_file(): + env_var_to_provider = {"ANTHROPIC_API_KEY": "anthropic", + "OPENAI_API_KEY": "openai"} + for line in env_file.read_text().splitlines(): + line = line.strip().removeprefix("export ").strip() + if not line or line.startswith("#") or "=" not in line: + continue + name, _, value = line.partition("=") + provider = env_var_to_provider.get(name.strip()) + if provider: + keys[provider] = value.strip().strip("'\"") + # --key-file entries override anything loaded from the env file + for spec in args.key_file: + if "=" in spec: + provider, _, path = spec.partition("=") + keys[provider] = Path(path).expanduser().read_text().strip() + else: + keys[""] = Path(spec).expanduser().read_text().strip() + env_key = os.environ.get("SKILL_SCANNER_LLM_API_KEY", "") + + def key_for(model: str) -> str: + provider = model.split("/", 1)[0] if "/" in model else "" + key = keys.get(provider) or keys.get("") or env_key + if not key: + sys.exit(f"No API key for {model}: pass --key-file {provider}= " + "or set SKILL_SCANNER_LLM_API_KEY") + return key + + for model in args.models: + key_for(model) # fail fast before any scans run + + if args.resume: + out_dir = Path(args.resume) + if not out_dir.is_dir(): + sys.exit(f"--resume dir not found: {out_dir}") + else: + stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + out_dir = Path(args.out) / stamp + out_dir.mkdir(parents=True, exist_ok=True) + cache_dir = out_dir.parent / "source-cache" + + results: dict[str, dict[str, dict]] = {} + for skill in args.skills: + meta = read_spec(skill) + source = checkout_source(skill, meta, cache_dir) + entries, _ = load_security_config(meta["spec_file"]) + results[skill] = {} + for model in args.models: + model_slug = model.replace("/", "_") + runs = [] + for i in range(args.runs): + scan_file = out_dir / f"{skill}--{model_slug}--run{i + 1}.json" + if scan_file.exists() and scan_file.stat().st_size > 0: + # Resumed: score the existing scan; wall-clock duration is + # lost, fall back to the scanner's (undercounting) figure. + print(f"[{skill}] {model} run {i + 1}/{args.runs} (cached)", flush=True) + scan_json = json.loads(scan_file.read_text()) + duration, ok = scan_json.get("scan_duration_seconds") or 0.0, True + else: + print(f"[{skill}] {model} run {i + 1}/{args.runs} ...", flush=True) + duration, ok = run_scan(source, scan_file, model, key_for(model), + args.temperature, args.consensus_runs) + rec = {"duration": duration, "ok": ok, + "blocking": None, "noise_high": None, "llm_keys": []} + if ok: + scan = json.loads(scan_file.read_text()) + blocking, _, _ = classify_findings(scan, entries) + no_allow, _, _ = classify_findings(scan, []) + usage = scan.get("llm_usage") or {} + rec.update({ + "blocking": len(blocking), + "noise_high": len(no_allow), + # analyzer field is "llm" or "static" (scanner 2.0.13) + "llm_keys": sort_keys(finding_keys(scan, {"llm"})), + "llm_input_tokens": usage.get("input_tokens"), + "llm_output_tokens": usage.get("output_tokens"), + }) + print(f" {duration:.0f}s ok={ok} blocking={rec['blocking']} " + f"noise_high={rec['noise_high']}", flush=True) + runs.append(rec) + results[skill][model] = summarize_cell(runs) + + (out_dir / "summary.json").write_text(json.dumps(results, indent=2)) + + lines = ["| Skill | Model | Mean time | Blocking/run | HIGH+ noise/run | LLM stability |", + "|---|---|---|---|---|---|"] + for skill, models in results.items(): + for model, cell in models.items(): + lines.append( + f"| {skill} | {model} | {cell['duration_mean_s']}s " + f"| {cell['blocking_per_run']} | {cell['noise_high_per_run']} " + f"| {cell['llm_stability_jaccard']} |") + md = "\n".join(lines) + "\n" + (out_dir / "summary.md").write_text(md) + print(f"\nResults written to {out_dir}\n\n{md}") + + +if __name__ == "__main__": + main() diff --git a/scripts/skill-scan/eval/bench_recall.py b/scripts/skill-scan/eval/bench_recall.py new file mode 100644 index 00000000..e3e05bc2 --- /dev/null +++ b/scripts/skill-scan/eval/bench_recall.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""Compare LLM models on skill-scanner's malicious and safe eval fixtures. + +Unlike skill-scanner 2.0.13's bundled benchmark runner, this script invokes +Dockyard's production run_scan.py wrapper, so the LLM and meta analyzers are +actually enabled. + +Usage: + python3 scripts/skill-scan/eval/bench_recall.py \ + --scanner-source /path/to/skill-scanner \ + --models anthropic/claude-sonnet-4-6 openai/gpt-5.6-terra \ + --runs 3 +""" + +import argparse +import datetime +import json +import os +import subprocess +import sys +from pathlib import Path + +from bench_models import EVAL_DIR, run_scan + +sys.path.insert(0, str(EVAL_DIR.parent)) +from process_scan_results import classify_findings # noqa: E402 + +BLOCKING_SEVERITIES = {"HIGH", "CRITICAL"} + + +def load_keys(key_files: list[str], env_file: Path) -> dict[str, str]: + keys: dict[str, str] = {} + if env_file.is_file(): + providers = {"ANTHROPIC_API_KEY": "anthropic", "OPENAI_API_KEY": "openai"} + for line in env_file.read_text().splitlines(): + line = line.strip().removeprefix("export ").strip() + if not line or line.startswith("#") or "=" not in line: + continue + name, _, value = line.partition("=") + provider = providers.get(name.strip()) + if provider: + keys[provider] = value.strip().strip("'\"") + for spec in key_files: + if "=" in spec: + provider, _, path = spec.partition("=") + keys[provider] = Path(path).expanduser().read_text().strip() + else: + keys[""] = Path(spec).expanduser().read_text().strip() + return keys + + +def key_for(model: str, keys: dict[str, str]) -> str: + provider = model.split("/", 1)[0] if "/" in model else "" + key = keys.get(provider) or keys.get("") or os.environ.get( + "SKILL_SCANNER_LLM_API_KEY", "" + ) + if not key: + raise SystemExit( + f"No API key for {model}: pass --key-file {provider}= " + "or set SKILL_SCANNER_LLM_API_KEY" + ) + return key + + +def discover_fixtures(scanner_source: Path) -> list[tuple[Path, Path]]: + fixture_root = scanner_source / "evals" / "skills" + if not fixture_root.is_dir(): + raise SystemExit(f"Fixture directory not found: {fixture_root}") + fixtures = [] + for expected_file in sorted(fixture_root.rglob("_expected.json")): + if (expected_file.parent / "SKILL.md").is_file(): + fixtures.append((expected_file.parent, expected_file)) + if not fixtures: + raise SystemExit(f"No evaluation fixtures found under {fixture_root}") + return fixtures + + +def source_revision(scanner_source: Path) -> str: + proc = subprocess.run( + ["git", "-C", str(scanner_source), "describe", "--always", "--tags", "--dirty"], + capture_output=True, + text=True, + ) + return proc.stdout.strip() if proc.returncode == 0 else "unknown" + + +def summarize(runs: list[dict]) -> dict: + ok = [run for run in runs if run["ok"]] + expected_slots = sum(run["expected_slots"] for run in ok) + return { + "runs_ok": len(ok), + "runs_total": len(runs), + "malicious_blocked": sum( + 1 for run in ok if not run["expected_safe"] and run["blocked"] + ), + "malicious_total": sum(1 for run in ok if not run["expected_safe"]), + "safe_clean": sum( + 1 for run in ok if run["expected_safe"] and not run["blocked"] + ), + "safe_total": sum(1 for run in ok if run["expected_safe"]), + "expected_slots_covered_any": sum(run["covered_any"] for run in ok), + "expected_slots_covered_high": sum(run["covered_high"] for run in ok), + "expected_slots_total": expected_slots, + "wall_time_seconds": round(sum(run["duration"] for run in runs), 1), + "input_tokens": sum(run["input_tokens"] for run in ok), + "output_tokens": sum(run["output_tokens"] for run in ok), + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--scanner-source", required=True) + parser.add_argument( + "--models", + nargs="+", + default=["anthropic/claude-sonnet-4-6", "openai/gpt-5.6-terra"], + ) + parser.add_argument("--runs", type=int, default=3) + parser.add_argument("--key-file", nargs="+", default=[]) + parser.add_argument("--env-file", default=str(EVAL_DIR / ".env")) + parser.add_argument("--temperature", default="0.0") + parser.add_argument("--out", default=str(EVAL_DIR / "results" / "recall")) + args = parser.parse_args() + + scanner_source = Path(args.scanner_source).expanduser().resolve() + fixtures = discover_fixtures(scanner_source) + keys = load_keys(args.key_file, Path(args.env_file).expanduser()) + for model in args.models: + key_for(model, keys) + + stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + out_dir = Path(args.out) / stamp + out_dir.mkdir(parents=True, exist_ok=True) + + all_runs: dict[str, list[dict]] = {model: [] for model in args.models} + for model in args.models: + model_slug = model.replace("/", "_") + for fixture, expected_file in fixtures: + expected = json.loads(expected_file.read_text()) + fixture_slug = "--".join(fixture.relative_to(scanner_source / "evals" / "skills").parts) + expected_categories = [ + finding.get("category") + for finding in expected.get("expected_findings", []) + if finding.get("category") + ] + for index in range(1, args.runs + 1): + output = out_dir / f"{fixture_slug}--{model_slug}--run{index}.json" + print(f"[{model}] {fixture_slug} run {index}/{args.runs} ...", flush=True) + duration, ok = run_scan( + fixture, + output, + model, + key_for(model, keys), + args.temperature, + None, + ) + record = { + "fixture": fixture_slug, + "expected_safe": expected.get("expected_safe", True), + "expected_slots": len(expected_categories), + "duration": duration, + "ok": ok, + "blocked": False, + "covered_any": 0, + "covered_high": 0, + "input_tokens": 0, + "output_tokens": 0, + } + if ok: + scan = json.loads(output.read_text()) + blocking, _, _ = classify_findings(scan, []) + actual_categories = { + finding.get("category") for finding in scan.get("findings", []) + } + high_categories = { + finding.get("category") + for finding in scan.get("findings", []) + if finding.get("severity") in BLOCKING_SEVERITIES + } + usage = scan.get("llm_usage") or {} + record.update( + { + "blocked": bool(blocking), + "covered_any": sum( + category in actual_categories for category in expected_categories + ), + "covered_high": sum( + category in high_categories for category in expected_categories + ), + "input_tokens": usage.get("input_tokens") or 0, + "output_tokens": usage.get("output_tokens") or 0, + } + ) + all_runs[model].append(record) + print( + f" {duration:.0f}s ok={ok} blocked={record['blocked']} " + f"coverage={record['covered_high']}/{record['expected_slots']} HIGH+", + flush=True, + ) + + summary = { + "scanner_source": str(scanner_source), + "scanner_revision": source_revision(scanner_source), + "runs_per_fixture": args.runs, + "fixtures": len(fixtures), + "models": {model: summarize(runs) for model, runs in all_runs.items()}, + } + (out_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + + lines = [ + "| Model | Malicious blocked | Safe clean | Expected coverage (any) " + "| Expected coverage (HIGH+) | Wall time | Input tokens | Output tokens |", + "|---|---:|---:|---:|---:|---:|---:|---:|", + ] + for model, data in summary["models"].items(): + lines.append( + f"| {model} | {data['malicious_blocked']}/{data['malicious_total']} " + f"| {data['safe_clean']}/{data['safe_total']} " + f"| {data['expected_slots_covered_any']}/{data['expected_slots_total']} " + f"| {data['expected_slots_covered_high']}/{data['expected_slots_total']} " + f"| {data['wall_time_seconds']}s | {data['input_tokens']} " + f"| {data['output_tokens']} |" + ) + markdown = "\n".join(lines) + "\n" + (out_dir / "summary.md").write_text(markdown) + print(f"\nResults written to {out_dir}\n\n{markdown}") + + +if __name__ == "__main__": + main() diff --git a/scripts/skill-scan/eval/merge_report.py b/scripts/skill-scan/eval/merge_report.py new file mode 100644 index 00000000..02fae78f --- /dev/null +++ b/scripts/skill-scan/eval/merge_report.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Merge bench_models candidate summaries + the mined CI baseline into one report. + +Usage: + python3 scripts/skill-scan/eval/merge_report.py \ + --baseline scripts/skill-scan/eval/results/ci-baseline/baseline-*.json \ + --candidates scripts/skill-scan/eval/results/m-*/2*/summary.json +""" + +import argparse +import glob +import json +import statistics +from pathlib import Path + +EVAL_DIR = Path(__file__).resolve().parent + + +def cell_stats(cell: dict) -> dict: + blocking = cell.get("blocking_per_run") or [] + noise = cell.get("noise_high_per_run") or [] + return { + "runs": len(blocking), + "block_rate": round(sum(1 for b in blocking if b) / len(blocking), 2) if blocking else None, + "blocking_mean": round(statistics.mean(blocking), 1) if blocking else None, + "noise_mean": round(statistics.mean(noise), 1) if noise else None, + "stability": cell.get("llm_stability_jaccard"), + "out_tokens": cell.get("llm_output_tokens_mean"), + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--baseline", default=None) + parser.add_argument("--candidates", nargs="+", default=None) + parser.add_argument("--baseline-label", default="sonnet-4-6 (CI)") + args = parser.parse_args() + + baseline_path = args.baseline or sorted( + glob.glob(str(EVAL_DIR / "results" / "ci-baseline" / "baseline-*.json")))[-1] + candidate_paths = args.candidates or sorted( + glob.glob(str(EVAL_DIR / "results" / "m-*" / "2*" / "summary.json"))) + + # skill -> model -> stats + table: dict[str, dict[str, dict]] = {} + models: list[str] = [args.baseline_label] + + # Only baseline scans from scanner 2.0.13 are comparable to the candidate + # runs (2.0.12 shipped different rule packs — e.g. codeql's ~144 HIGH+ + # noise under 2.0.12 vs ~2 under 2.0.13). Artifacts predating the + # scanner-version.txt upload report "" — classify those by date: + # the 2.0.13 pin merged 2026-08-11 (commit dfd663b). + V213_CUTOFF = "2026-08-12" + + def is_v213(run: dict) -> bool: + v = run.get("scanner_version") or "" + if v: + return v == "2.0.13" + return run.get("artifact_created", "") >= V213_CUTOFF + + baseline = json.loads(Path(baseline_path).read_text()) + for skill, data in baseline.items(): + for ref, group in data["groups"].items(): + if not group["is_current_ref"]: + continue + runs = [r for r in group.get("runs", []) if is_v213(r)] + if not runs: + continue + llm_sets = [set(map(tuple, r["llm_keys"])) for r in runs] + import itertools + sims = [len(a & b) / len(a | b) if (a or b) else 1.0 + for a, b in itertools.combinations(llm_sets, 2)] + toks = [r["llm_output_tokens"] for r in runs if r.get("llm_output_tokens")] + table.setdefault(skill, {})[args.baseline_label] = cell_stats({ + "blocking_per_run": [r["blocking"] for r in runs], + "noise_high_per_run": [r["noise_high"] for r in runs], + "llm_stability_jaccard": round(statistics.mean(sims), 3) if sims else None, + "llm_output_tokens_mean": round(statistics.mean(toks)) if toks else None, + }) + + for path in candidate_paths: + summary = json.loads(Path(path).read_text()) + # Label by results directory (m-