From 2a35b57a8af5417e0a4fdce76dc341136389a88c Mon Sep 17 00:00:00 2001 From: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:37:51 -0700 Subject: [PATCH 01/13] [None][feat] Add a modeling-task-yaml skill for agent-flow task briefs The skill builds an agent-flow modeling task.yaml (modeling-bringup or modeling-v2-onboard) through a conversation: it confirms the known fields, asks about the important ones the user has not mentioned, and infers what it can from the checkpoint before asking the user to confirm it. inspect_checkpoint.py reads a checkpoint's facts, and validate_task_yaml.py checks the finished brief against the schema. Signed-off-by: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> --- .claude/skills/modeling-task-yaml/SKILL.md | 250 ++++++++++++++++++ .../scripts/inspect_checkpoint.py | 118 +++++++++ .../scripts/validate_task_yaml.py | 88 ++++++ ...6-10-25-modeling-task-yaml-skill-design.md | 159 +++++++++++ 4 files changed, 615 insertions(+) create mode 100644 .claude/skills/modeling-task-yaml/SKILL.md create mode 100644 .claude/skills/modeling-task-yaml/scripts/inspect_checkpoint.py create mode 100644 .claude/skills/modeling-task-yaml/scripts/validate_task_yaml.py create mode 100644 docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md diff --git a/.claude/skills/modeling-task-yaml/SKILL.md b/.claude/skills/modeling-task-yaml/SKILL.md new file mode 100644 index 000000000000..14e2b1c8600a --- /dev/null +++ b/.claude/skills/modeling-task-yaml/SKILL.md @@ -0,0 +1,250 @@ +--- +name: modeling-task-yaml +description: Use when a user wants to write, generate, or fill in a task.yaml (task brief) for an agent-flow modeling workflow (modeling-bringup or modeling-v2-onboard), or asks how to onboard / bring up a new model with agent-flow. +license: Apache-2.0 +metadata: + author: NVIDIA Corporation +--- + +# Modeling task.yaml + +Build an agent-flow modeling `task.yaml` through a conversation. The schema is +thin (three paths, optional `slurm-environment`, and two free-text lists), so +what makes a run succeed — the accuracy bars, the features to prove, the +environment — lives in sentences under `completion_criteria` and +`implements_tips`. Your job is to ask for what the user left out and turn +the answers into clear, checkable sentences. + +## Conversation contract + +- Talk in the user's language; write the YAML sentences in English. +- Part 1 and Part 2 are each asked as one form (see below), one part per + message; Parts 3 and 4 go one question per message. Use AskUserQuestion + for anything with fixed choices (the feature checklist, yes/no); without + it, list the choices in plain text. +- Show values, not verdicts. Whenever you propose, infer, or recap values, + list every field with its exact value (full paths, names, numbers) and, + where you checked it, the check result. "All verified" or "already filled + in" without the values is never enough. +- Never put a value in the YAML the user has not seen. Values you take from + anywhere other than the user's own words (earlier task.yaml files, the + environment, the checkpoint) are shown with their source and confirmed. +- Say every default out loud before you use it. +- End each part with a recap of what goes into the YAML, in the same + field-by-value form, and ask for corrections before moving on. +- Write only what this skill lists. Do not add environment variables + (PATH, PYTHONPATH, HOME, HF offline, MPI/UCX), build-artifact symlinks, or + `gpu_pool`. If the user volunteers such details, copy them verbatim into + `implements_tips`. + +## Forms for Parts 1 and 2 + +Part 1 and Part 2 each get one form: a single message asking for every item +of that part as a numbered list the user answers in one reply. Part 2's form +is sent only after Part 1 is confirmed. + +Before each form, look at what is at hand: the user's messages, +`nvidia-smi`, the current checkout, the checkpoint directory (e.g. a +`modeling_*.py` in it). Pre-fill each value you found, written out in full +with its source (e.g. "from your message", "from nvidia-smi", "found in the +checkpoint directory"); leave the rest blank. In Part 2's form, the Slurm +fields are answered only if the user answers yes to "use Slurm?". + +When the reply arrives, run that part's checks (and, in Part 1, the +checkpoint inspection), then send one recap: a table of every field, its +value, its source, and its check result (Part 1 adds the checkpoint facts). +Ask about blanks, failed checks and conflicting values; otherwise ask the +user to confirm the table. + +If the user points to an earlier task.yaml, take from it only the fields +this skill writes. Map `slurm-environment.slurm_partition` and +`docker_image` directly, and `gpu_pool.account` / `qos` / +`container_mounts` to the same names under `slurm-environment`. When the +file holds two values for one field (e.g. `slurm_partition` and +`gpu_pool.partition`), show both and ask. Its other content (environment +variables, other tips) is not carried over unless the user asks for it. + +## Part 1 — Basic configuration + +Items and checks: + +| Item | Goes to | Check | +|---|---|---| +| `reference_code_path` (HF `modeling_*.py`) | top level | exists | +| `checkpoint_path` (HF checkpoint dir) | top level | exists, has `config.json` | +| `trtllm_repo_path` | top level | exists, is a git checkout (`git -C rev-parse`) | +| Target GPU (e.g. GB300 / sm_103) | `implements_tips` | if `nvidia-smi` works here, propose its GPU and confirm; otherwise ask — never guess the model from a partition name. You may fill in the SM from the GPU name; show it for confirmation | +| `LLM_MODELS_ROOT` | `implements_tips` | directory exists; `datasets/openai/gsm8k` exists and is non-empty — warn if missing | + +A path that does not exist goes back to the user; never write it. + +Inspect the checkpoint with: + +```bash +python3 .claude/skills/modeling-task-yaml/scripts/inspect_checkpoint.py +``` + +Show what it found (architecture, dtype, multimodal, MTP layers, +quantization, weight size, chat template) in the recap. Parts 3 and 4 use +these facts. + +## Part 2 — Run mode + +Whether the run uses Slurm: + +- **No:** omit `slurm-environment` entirely. +- **Yes:** these five go under `slurm-environment` + (never under `gpu_pool`, which starts a node pool in modeling-v2-onboard). + The agents read this whole section. + + | Field | Required | Check | + |---|---|---| + | `slurm_partition` | yes | — | + | `docker_image` | yes | the `.sqsh` exists | + | `account` | no — omit if the user has none | — | + | `qos` | no — omit if the user has none | — | + | `container_mounts` | no — a list of paths | each path exists; together they cover `trtllm_repo_path`, `checkpoint_path`, `reference_code_path` and `LLM_MODELS_ROOT` — name any path left uncovered and ask whether to add a mount | + +## Part 3 — Accuracy + +**End-to-end accuracy.** Default: GSM8K via `trtllm-eval`, within 1 point +(absolute) of the reference score. The user may change the dataset or the +tolerance. Ask whether they already have a reference score. If not, decide +whether HF can run the checkpoint (below), tell the user the verdict and +why, and ask them to confirm it. + +```dot +digraph reference { + "User has a reference score?" [shape=diamond]; + "HF can run this checkpoint?" [shape=diamond]; + "Write the score into the criterion" [shape=box]; + "Agent measures the HF reference" [shape=box]; + "Explain why; require a score or another baseline" [shape=box]; + "User has a reference score?" -> "Write the score into the criterion" [label="yes"]; + "User has a reference score?" -> "HF can run this checkpoint?" [label="no"]; + "HF can run this checkpoint?" -> "Agent measures the HF reference" [label="yes"]; + "HF can run this checkpoint?" -> "Explain why; require a score or another baseline" [label="no"]; +} +``` + +HF cannot run the checkpoint when its quantization is a TRT-LLM-specific +format (e.g. NVFP4, ModelOpt FP8 from `hf_quant_config.json`) or when +`weight_size_gb` exceeds about 80% of the total GPU memory of one node (HF +spreads one model over a node's GPUs with `device_map="auto"`; it does not +span nodes). The other baseline can be a +BF16 checkpoint of the same model (ask for its path and check it) or a +published score (ask for the number and its source). Never write a +criterion the run cannot reach. + +If the checkpoint has a chat template, add the chat-template tip below. + +**Single-op accuracy.** Always add the op criterion below, unchanged. + +## Part 4 — Features + +Ask with one multi-select question: + +| Feature | Default | +|---|---| +| CUDA graph | on | +| overlap scheduler | on | +| chunked prefill | on | +| KV cache reuse | off | +| speculative decoding | off | +| disaggregated serving | off | +| multimodal (vision part) | off; offer only when Part 1 found the model multimodal | + +- All chosen features except disagg and multimodal go into one criterion, + named as in the table: the end-to-end accuracy bar is met with all of them + on together. If none are chosen, omit that criterion. +- **Speculative decoding / disagg:** ask each once for optional notes + (algorithm, draft model, topology). With notes, put them in + `implements_tips`; without, add the "left to the planner" tip. When the + checkpoint has MTP layers, tell the user before asking for spec-decoding + notes, and use the MTP variant of the tip. +- **Disagg** gets its own criterion: `trtllm-eval` runs the model + in-process and cannot drive a disaggregated server. +- **Multimodal chosen:** add MMMU with the same rules as GSM8K (reference + score or measured or required baseline, same tolerance, same feature set). + Check now that `LLM_MODELS_ROOT/datasets/MMMU` exists and is non-empty, + and warn if not. + **Multimodal model but not chosen:** add the text-only tip. + +## Finish + +1. Show the complete YAML and ask for changes. +2. Ask for the workspace directory; write `/task.yaml`. If the + directory does not exist, say so and create it once the user agrees. If + the file exists, ask before overwriting. +3. Validate with agent-flow's own schema, fix, and re-run until it passes: + ```bash + python3 .claude/skills/modeling-task-yaml/scripts/validate_task_yaml.py /task.yaml + ``` +4. Give the run command: + `modeling-bringup --task /task.yaml --workspace `. + If `modeling-bringup` is not on PATH, add the install step + `pip install -e /agent-flow`. Mention + `modeling-v2-onboard` (same arguments) only when the user's agent-flow + has `agent_flow/workflows/modeling_v2_onboard/`. + +## Sentence library + +Fill the `<...>` slots; keep the wording. + +`completion_criteria`: + +- E2E, known score: `" via trtllm-eval scores within point(s) (absolute) of the reference score ()."` +- E2E, measured: `" via trtllm-eval scores within point(s) (absolute) of the HuggingFace implementation at reference_code_path, measured by the agent under the same evaluation settings (dataset split, prompt format, chat template, sampling)."` +- E2E, other baseline: `" via trtllm-eval scores within point(s) (absolute) of ()."` +- Op: `"Every op matches its reference within 1 ulp of the output dtype (relative to each row's max magnitude). A wider band needs a stated derivation. A faster replacement need not match the previous op bit for bit."` +- Features: `"The accuracy bar above is met with all enabled together."` +- Disagg: `" served through a disaggregated trtllm-serve deployment (separate context and generation servers) meets the same accuracy bar."` + +`implements_tips`: + +- `"Target GPU: ()."` +- `"LLM_MODELS_ROOT=; GSM8K is under datasets/openai/gsm8k<; MMMU under datasets/MMMU>."` +- Chat template: `"The checkpoint ships a chat template (); decide whether evaluation must apply it and use the same choice for the reference."` +- Spec/disagg notes: `": ."` +- Spec/disagg, no notes: `": the algorithm and configuration are left to the planner to infer from the checkpoint and the codebase."` +- Spec, no notes, MTP checkpoint: `"Speculative decoding: the checkpoint has MTP layer(s) (); the algorithm and configuration are left to the planner to infer from the checkpoint and the codebase."` +- Text only: `"Text model only; the vision tower and projector are out of scope."` + +## Output shape + +```yaml +reference_code_path: /path/to/modeling_.py +checkpoint_path: /path/to/ +trtllm_repo_path: /path/to/TensorRT-LLM + +slurm-environment: # only when the user runs on Slurm + slurm_partition: + docker_image: /path/to/image.sqsh + account: + qos: + container_mounts: + - /path/to/mount + +completion_criteria: + - "" + - "" + - "" + - "" + +implements_tips: + - "" + - "" + - "" +``` + +## Common mistakes + +| Mistake | Fix | +|---|---| +| Writing a default the user never heard | Say it, then write it | +| Asking Part 1 or Part 2 one field at a time, or merging both into one form | One form per part, one reply each | +| "Slurm config inferred and verified, please confirm" with no values shown | List every field with its value, source and check result | +| Folding disagg into the `trtllm-eval` feature criterion | Disagg gets its own criterion | +| "Agent measures the HF reference" for an NVFP4 / oversized checkpoint | Require a score or another baseline | +| Putting account / qos / mounts under `gpu_pool` | Directly under `slurm-environment` | +| Importing `agent_flow` to validate | Use `scripts/validate_task_yaml.py` | diff --git a/.claude/skills/modeling-task-yaml/scripts/inspect_checkpoint.py b/.claude/skills/modeling-task-yaml/scripts/inspect_checkpoint.py new file mode 100644 index 000000000000..4bb0b8bc0002 --- /dev/null +++ b/.claude/skills/modeling-task-yaml/scripts/inspect_checkpoint.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Summarize the facts about a HuggingFace checkpoint that shape a task.yaml. + +Prints one JSON object: architectures, dtype, whether the model is +multimodal, its MTP layer count, its quantization, its weight size, and +whether it ships a chat template. Every field is read from files in the +checkpoint directory; nothing is imported or downloaded. + +Usage: + python3 inspect_checkpoint.py /path/to/checkpoint +""" + +import argparse +import json +import sys +from pathlib import Path + +MULTIMODAL_CONFIG_KEYS = ("vision_config", "audio_config", "visual", "vision_tower") +PROCESSOR_FILES = ("preprocessor_config.json", "processor_config.json") +MTP_KEYS = ("num_nextn_predict_layers", "num_mtp_layers", "mtp_num_layers") +WEIGHT_SUFFIXES = (".safetensors", ".bin", ".pt") + + +def _find_key(config: dict, keys: tuple[str, ...]): + """Return (key, value) of the first match, looking in text_config too.""" + for scope in (config, config.get("text_config") or {}): + for key in keys: + if key in scope: + return key, scope[key] + return None, None + + +def _quantization(ckpt: Path, config: dict): + quant = config.get("quantization_config") + if quant: + return { + "source": "config.json:quantization_config", + "method": quant.get("quant_method") or quant.get("quant_algo"), + "detail": quant, + } + modelopt = ckpt / "hf_quant_config.json" + if modelopt.is_file(): + detail = json.loads(modelopt.read_text()) + algo = (detail.get("quantization") or {}).get("quant_algo") + return {"source": "hf_quant_config.json", "method": algo, "detail": detail} + return None + + +def _chat_template(ckpt: Path): + if (ckpt / "chat_template.jinja").is_file(): + return "chat_template.jinja" + tokenizer_config = ckpt / "tokenizer_config.json" + if tokenizer_config.is_file(): + if json.loads(tokenizer_config.read_text()).get("chat_template"): + return "tokenizer_config.json:chat_template" + return None + + +def inspect(ckpt: Path) -> dict: + config_path = ckpt / "config.json" + if not config_path.is_file(): + raise FileNotFoundError(f"no config.json in {ckpt}") + config = json.loads(config_path.read_text()) + + multimodal_evidence = [k for k in MULTIMODAL_CONFIG_KEYS if k in config] + multimodal_evidence += [f for f in PROCESSOR_FILES if (ckpt / f).is_file()] + + weight_bytes = sum( + p.stat().st_size for p in ckpt.iterdir() if p.is_file() and p.suffix in WEIGHT_SUFFIXES + ) + + mtp_key, mtp_layers = _find_key(config, MTP_KEYS) + + return { + "checkpoint_path": str(ckpt), + "architectures": config.get("architectures"), + "model_type": config.get("model_type"), + "dtype": _find_key(config, ("torch_dtype", "dtype"))[1], + "multimodal": bool(multimodal_evidence), + "multimodal_evidence": multimodal_evidence, + "mtp_layers": mtp_layers, + "mtp_config_key": mtp_key, + "quantization": _quantization(ckpt, config), + "weight_size_gb": round(weight_bytes / 1e9, 1), + "chat_template": _chat_template(ckpt), + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("checkpoint", type=Path, help="HF checkpoint directory") + args = parser.parse_args() + try: + summary = inspect(args.checkpoint) + except (FileNotFoundError, json.JSONDecodeError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + print(json.dumps(summary, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.claude/skills/modeling-task-yaml/scripts/validate_task_yaml.py b/.claude/skills/modeling-task-yaml/scripts/validate_task_yaml.py new file mode 100644 index 000000000000..57aeaebba6f6 --- /dev/null +++ b/.claude/skills/modeling-task-yaml/scripts/validate_task_yaml.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Validate a task.yaml with agent-flow's own modeling-bringup schema. + +Loads agent-flow's task_schema.py straight from its file: importing it +through the agent_flow package also imports claude_agent_sdk, which is not +installed everywhere. task_schema.py itself needs only PyYAML. + +Usage: + python3 validate_task_yaml.py /path/to/task.yaml [--agent-flow DIR] + +Without --agent-flow, the schema is looked up under the task's +trtllm_repo_path, then under this checkout. +""" + +import argparse +import importlib.util +import sys +from pathlib import Path + +import yaml + +SCHEMA_RELPATH = Path("agent_flow/workflows/modeling_bringup/task_schema.py") +THIS_CHECKOUT = Path(__file__).resolve().parents[4] + + +def _candidate_agent_flow_dirs(task_path: Path, explicit: Path | None) -> list[Path]: + if explicit is not None: + return [explicit] + candidates = [] + try: + data = yaml.safe_load(task_path.read_text(encoding="utf-8")) or {} + except (OSError, yaml.YAMLError): + data = {} + repo = data.get("trtllm_repo_path") if isinstance(data, dict) else None + if isinstance(repo, str) and repo.strip(): + candidates.append(Path(repo) / "agent-flow") + candidates.append(THIS_CHECKOUT / "agent-flow") + return candidates + + +def _load_schema_module(agent_flow_dir: Path): + schema_file = agent_flow_dir / SCHEMA_RELPATH + spec = importlib.util.spec_from_file_location("task_schema", schema_file) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("task", type=Path, help="task.yaml to validate") + parser.add_argument("--agent-flow", type=Path, help="agent-flow directory holding the schema") + args = parser.parse_args() + + candidates = _candidate_agent_flow_dirs(args.task, args.agent_flow) + agent_flow_dir = next((d for d in candidates if (d / SCHEMA_RELPATH).is_file()), None) + if agent_flow_dir is None: + tried = ", ".join(str(d / SCHEMA_RELPATH) for d in candidates) + print(f"error: task_schema.py not found; tried: {tried}", file=sys.stderr) + return 2 + + schema = _load_schema_module(agent_flow_dir) + try: + schema.load_and_validate_task_yaml(args.task) + except schema.TaskSchemaError as exc: + print(f"INVALID\n{exc}") + return 1 + print(f"VALID ({agent_flow_dir / SCHEMA_RELPATH})") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md b/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md new file mode 100644 index 000000000000..01473e8515ba --- /dev/null +++ b/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md @@ -0,0 +1,159 @@ +# 设计:交互式生成 agent-flow modeling task.yaml 的 skill + +## 目的与动机 + +在 agent-flow 中接入一个新模型时,用户需要手写 `task.yaml`。schema 本身很简单:三个路径、 +可选的 `slurm-environment`、两个自由文本列表 `completion_criteria` 和 `implements_tips`。 +真正决定一次运行成败的内容,比如精度标准、要验证哪些功能、环境信息,都只能靠用户用自然语言 +写进这两个列表。这些内容容易漏写,也容易写得模糊。 + +这个 skill 通过对话引导用户填写: +- 已知字段逐项确认; +- 用户没有主动提到、但很重要的字段,主动询问; +- 能从 checkpoint 推断出来的信息,先推断再请用户确认; + +最后产出一份能直接交给工作流运行的 `task.yaml`。 + +## 目标 + +- 覆盖 `modeling-bringup` 和 `modeling-v2-onboard` 共用的 task.yaml 部分。两者共用 + `agent-flow/agent_flow/workflows/modeling_bringup/task_schema.py`,该文件已在 main 上。 +- 分四个部分对话:基础配置 → 运行方式 → 精度验证 → 功能验证。 +- 把用户的选择展开成清晰、可验证的英文句子,写进 `completion_criteria` 和 `implements_tips`。 +- 写出文件后,用 `load_and_validate_task_yaml` 校验,并告诉用户怎样运行。 + +## 非目标 + +- `modeling-v2-onboard` 特有的内容:`gpu_pool`(节点池)、并行拓扑选择、性能 Pareto、性能模型。 + 这些等 v2-onboard 合入 main 后再扩展。 +- 修改 agent-flow 的 schema 或工作流代码。 +- 不问、也不写入环境细节:PATH/PYTHONPATH/HOME、HF 离线变量、MPI/UCX 变量、编译产物软链接。 + 用户主动提供时,可以原样写进 `implements_tips`。 + +## 反目标(技术上能用,但仍算失败) + +- 悄悄填入用户没听说过的默认值。所有默认值都必须在对话里说出来。 +- 一次抛出一长串问题,让用户像填表一样回答。 +- 生成的 task.yaml 通不过 schema 校验,或者里面的路径不存在。 + +## Skill 位置与形式 + +- 位置:`.claude/skills/modeling-task-yaml/SKILL.md`。名字按 `.claude/README.md` 的 + `-` 规则取。(我的决定) +- `SKILL.md` 描述对话流程。可以附带一份带注释的模板,或者一份标准句子库,用来存放下文 + 要写入的英文句子。 +- schema 校验直接调用 agent-flow 里现有的 `load_and_validate_task_yaml`,不要另写一份 + schema。导入方式:直接按文件路径加载 `task_schema.py`,例如用 `importlib` 从文件加载。 + 原因是通过 `agent_flow` 包导入会连带导入 `claude_agent_sdk`,而这个包在部分机器上 + 不可用。(我的决定) +- 验证方式:拿一个真实 checkpoint 手动走一遍对话流程,不加自动化测试。(我的决定) + +## 对话流程 + +总的原则: +- 第 1、2 部分各用一张表单,一条消息问完这一部分的所有字段;第 3、4 部分一次只问一件事。 +- 有固定候选的问题,用选择题,即 AskUserQuestion。 +- 每个部分结束时,用表格逐字段列出具体值、来源和检查结果,请用户确认;不能只说"已核实"。 + +### 第 1 部分:基础配置 + +| 项 | 写入位置 | 检查 | +|---|---|---| +| `reference_code_path`(HF modeling 文件) | 顶层 | 必须存在 | +| `checkpoint_path`(HF checkpoint 目录) | 顶层 | 必须存在,并且有 `config.json` | +| `trtllm_repo_path` | 顶层 | 必须存在,并且是 git checkout | +| 目标 GPU(例如 GB300 / sm_103) | `implements_tips` 的一句话 | 能从 partition 或本机推断时,先推断再确认 | +| `LLM_MODELS_ROOT` | `implements_tips` 的一句话 | 目录必须存在;检查下面有没有 `datasets/openai/gsm8k`,选了多模态时还要检查 MMMU 数据,没有就提醒用户 | + +拿到 checkpoint 之后,读取 `config.json` 等文件,推断以下信息,供后面几部分使用: +- 架构名; +- 是否多模态,依据包括 `vision_config`、`preprocessor_config.json` 等; +- 是否带 MTP 层,例如 `num_nextn_predict_layers`; +- 是否有 `chat_template.jinja`。 + +推断出来的结论要说给用户听。 + +### 第 2 部分:运行方式 + +先问"是否使用 slurm 模式"。 + +- 不用:不写 `slurm-environment`。 +- 用:问以下 5 项,全部写在 `slurm-environment` 下面,不写 `gpu_pool`。 + - `slurm_partition` + - `docker_image`:`.sqsh` 文件必须存在 + - `account` + - `qos` + - `container_mounts` + +`modeling-bringup` 的 prompt 会让 agent 读取整段 `slurm-environment`,所以 account、qos、 +`container_mounts` 写在这里,agent 能看到。 + +### 第 3 部分:精度验证 + +**e2e 精度** +- 默认用 GSM8K,通过 `trtllm-eval` 跑。标准是和 reference 分数相差小于 1%(绝对分差)。 +- 先问用户有没有 reference 分数: + - 有:把具体数值写进标准。 + - 没有:先判断 HF 实现能不能跑这个 checkpoint,依据是 `config.json` 里的 + `quantization_config`(例如 NVFP4 这类 TRT-LLM 专用格式 HF 跑不了)和权重大小 + (能否在目标节点上用 HF 单进程跑)。 + - 能跑:写成"agent 用 HF 实现(`reference_code_path`),在同样的评测设置下自己跑出 + reference 分数,再和它比较"。 + - 跑不了:把原因告诉用户,并要求用户提供以下任意一种:一个具体分数;或者另一个基准, + 比如同一模型的 BF16 checkpoint 路径,或者公开发布的分数。不能写一条达不到的标准。 +- 用户可以换数据集,也可以改容忍度。 +- 如果第 4 部分勾选了多模态,再加 MMMU,规则和 GSM8K 完全相同。 +- 如果存在 `chat_template.jinja`,在 `implements_tips` 里提醒:评测时要确认是否需要套用 + chat template。 + +**单 op 精度**:默认写入 `completion_criteria`,原文就用这句: + +> "Every op matches its reference within 1 ulp of the output dtype (relative to each row's +> max magnitude). A wider band needs a stated derivation. A faster replacement need not match +> the previous op bit for bit." + +### 第 4 部分:功能验证 + +用多选题,候选如下: + +| 功能 | 默认 | +|---|---| +| CUDA graph | 勾选 | +| overlap scheduler | 勾选 | +| chunked prefill | 勾选 | +| KV cache reuse | 不勾选 | +| speculative decoding | 不勾选 | +| disaggregated serving | 不勾选 | +| 多模态(视觉部分) | 不勾选;只有第 1 部分识别为多模态时才出现 | + +- **验证方式**:所有勾选的功能同时打开,在这个配置下跑第 3 部分的 e2e 精度验证 + (GSM8K,勾选多模态时还有 MMMU),并达到标准。写成一条 `completion_criteria`, + 其中列出全部勾选的功能。 +- **disagg 例外**:`trtllm-eval` 在进程内跑模型,不支持 disagg。所以 disagg 不放进上面那次 + 合并运行,单独写一条标准:"GSM8K(勾选多模态时还有 MMMU)通过 `trtllm-serve` 的 disagg + 服务来跑,标准和 e2e 精度相同"。服务怎么搭建交给 planner 推断。 +- **spec decoding / disagg**:勾选后,问一句"有没有要补充的说明",用户可以不填。 + - 填了:把说明写进 `implements_tips`。 + - 没填:写一句 "left to the planner to infer from the checkpoint and the codebase"。 + - checkpoint 带 MTP 层时,可以把这个事实告诉用户,作为参考。 +- **多模态**: + - 勾选:精度标准加上 MMMU。 + - 识别为多模态但没有勾选:在 `implements_tips` 里写明 + "text model only; vision tower out of scope"。 + +### 收尾 + +1. 展示完整的 task.yaml 草稿,请用户确认或修改。 +2. 问 workspace 路径,把 `task.yaml` 写进去。如果目标文件已经存在,先问用户,不能直接覆盖。 +3. 用 agent-flow 的 `load_and_validate_task_yaml` 校验。校验失败时,把错误告诉用户, + 改完再校验。 +4. 告诉用户运行命令,例如 `modeling-bringup --task --workspace `。 + +## 交给实现者决定的事 + +- skill 的具体名字和 frontmatter 里的 description 怎么写。description 要让 Claude 在 + "写 / 生成 modeling task.yaml、接入新模型到 agent-flow" 这类请求下触发。 +- 标准句子的最终英文措辞。除单 op 精度那句以外,其余句子保持精简、可以验证即可。 +- 用模板文件还是在 SKILL.md 里写示例。 +- 怎样找到 agent-flow 里的 `task_schema.py`:优先用 `trtllm_repo_path` 下的 + `agent-flow/`,找不到时再问用户。 From f15f35bdc6e31593e8962c5d2305156c6a545c73 Mon Sep 17 00:00:00 2001 From: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> Date: Fri, 6 Nov 2026 02:30:49 -0800 Subject: [PATCH 02/13] [None][feat] modeling-task-yaml: leave checks to the workflow, ask about the GPU pool and datasets The skill no longer checks paths, the image or mounts in the conversation: modeling-v2-onboard --check does it at Finish (validate_task_yaml.py when the agent-flow has no modeling-v2-onboard), and the runtime --smoke check is offered as an option. On Slurm it asks whether to use a GPU pool; the user gives nodes and max_nodes and the skill infers and confirms the rest. LLM_MODELS_ROOT is no longer asked: the dataset comes from a local path the user gives or is downloaded by the agent. Signed-off-by: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> --- .claude/skills/modeling-task-yaml/SKILL.md | 147 ++++++++++++------ ...6-10-25-modeling-task-yaml-skill-design.md | 50 +++--- 2 files changed, 127 insertions(+), 70 deletions(-) diff --git a/.claude/skills/modeling-task-yaml/SKILL.md b/.claude/skills/modeling-task-yaml/SKILL.md index 14e2b1c8600a..674012c84b8a 100644 --- a/.claude/skills/modeling-task-yaml/SKILL.md +++ b/.claude/skills/modeling-task-yaml/SKILL.md @@ -33,9 +33,12 @@ the answers into clear, checkable sentences. - End each part with a recap of what goes into the YAML, in the same field-by-value form, and ask for corrections before moving on. - Write only what this skill lists. Do not add environment variables - (PATH, PYTHONPATH, HOME, HF offline, MPI/UCX), build-artifact symlinks, or - `gpu_pool`. If the user volunteers such details, copy them verbatim into + (PATH, PYTHONPATH, HOME, HF offline, MPI/UCX) or build-artifact symlinks. + If the user volunteers such details, copy them verbatim into `implements_tips`. +- Check paths and settings once, at Finish, with the workflow's own check; + do not re-implement its checks in the conversation. A problem it reports + goes back to the user. ## Forms for Parts 1 and 2 @@ -50,33 +53,27 @@ with its source (e.g. "from your message", "from nvidia-smi", "found in the checkpoint directory"); leave the rest blank. In Part 2's form, the Slurm fields are answered only if the user answers yes to "use Slurm?". -When the reply arrives, run that part's checks (and, in Part 1, the -checkpoint inspection), then send one recap: a table of every field, its -value, its source, and its check result (Part 1 adds the checkpoint facts). -Ask about blanks, failed checks and conflicting values; otherwise ask the -user to confirm the table. +When the reply arrives (and, in Part 1, after the checkpoint inspection), +send one recap: a table of every field, its value and its source (Part 1 +adds the checkpoint facts). Ask about blanks and conflicting values; +otherwise ask the user to confirm the table. If the user points to an earlier task.yaml, take from it only the fields -this skill writes. Map `slurm-environment.slurm_partition` and -`docker_image` directly, and `gpu_pool.account` / `qos` / -`container_mounts` to the same names under `slurm-environment`. When the -file holds two values for one field (e.g. `slurm_partition` and -`gpu_pool.partition`), show both and ask. Its other content (environment +this skill writes, and propose them as values with that file as their +source. When the file holds two values for one field (e.g. `slurm_partition` +and `gpu_pool.partition`), show both and ask. Its other content (environment variables, other tips) is not carried over unless the user asks for it. ## Part 1 — Basic configuration Items and checks: -| Item | Goes to | Check | +| Item | Goes to | Notes | |---|---|---| -| `reference_code_path` (HF `modeling_*.py`) | top level | exists | -| `checkpoint_path` (HF checkpoint dir) | top level | exists, has `config.json` | -| `trtllm_repo_path` | top level | exists, is a git checkout (`git -C rev-parse`) | +| `reference_code_path` (HF `modeling_*.py`) | top level | — | +| `checkpoint_path` (HF checkpoint dir) | top level | — | +| `trtllm_repo_path` | top level | a clean checkout; modeling-v2-onboard commits its result there | | Target GPU (e.g. GB300 / sm_103) | `implements_tips` | if `nvidia-smi` works here, propose its GPU and confirm; otherwise ask — never guess the model from a partition name. You may fill in the SM from the GPU name; show it for confirmation | -| `LLM_MODELS_ROOT` | `implements_tips` | directory exists; `datasets/openai/gsm8k` exists and is non-empty — warn if missing | - -A path that does not exist goes back to the user; never write it. Inspect the checkpoint with: @@ -93,23 +90,54 @@ these facts. Whether the run uses Slurm: - **No:** omit `slurm-environment` entirely. -- **Yes:** these five go under `slurm-environment` - (never under `gpu_pool`, which starts a node pool in modeling-v2-onboard). - The agents read this whole section. - - | Field | Required | Check | - |---|---|---| - | `slurm_partition` | yes | — | - | `docker_image` | yes | the `.sqsh` exists | - | `account` | no — omit if the user has none | — | - | `qos` | no — omit if the user has none | — | - | `container_mounts` | no — a list of paths | each path exists; together they cover `trtllm_repo_path`, `checkpoint_path`, `reference_code_path` and `LLM_MODELS_ROOT` — name any path left uncovered and ask whether to add a mount | +- **Yes:** ask for `slurm_partition` and `docker_image` (both required, + under `slurm-environment`), and — only when the user's agent-flow has + `agent_flow/workflows/modeling_v2_onboard/` — whether the run uses a + **GPU pool**: a set of whole nodes the run holds from its start and + shares among its workers, instead of each worker submitting its own jobs. + +**Without a pool**, these optional fields also go directly under +`slurm-environment`; the agents read them: + +| Field | Value | +|---|---| +| `account` | omit if the user has none | +| `qos` | omit if the user has none | +| `container_mounts` | a list of paths; together with the checkout they cover `checkpoint_path` and `reference_code_path` | + +**With a pool**, they go under `slurm-environment.gpu_pool` instead, never +in both places. The user gives only `nodes` (holder jobs submitted at the +start) and `max_nodes` (the most the planner may grow it to; omit for the +worker count). Fill in the rest yourself from this environment, Slurm +(`sacctmgr`, `sinfo`), an earlier task.yaml or your memory, and show each +with its source for confirmation: + +| Field | Value | +|---|---| +| `account` | required | +| `qos` | omit if the user's association has none to pick | +| `gpus_per_node` | the partition's GPUs per node | +| `hours` | a holder job's time limit: 24, or less if the partition or QOS allows less | +| `container_mounts` | the fewest existing directories that hold `checkpoint_path`, `reference_code_path` and any local dataset path (Part 3); the workspace and the checkout are mounted without being listed | + +Say the defaults you leave out: `partition` is `slurm_partition`, and +`reserve` keeps 2 free GPUs from perf work. ## Part 3 — Accuracy **End-to-end accuracy.** Default: GSM8K via `trtllm-eval`, within 1 point (absolute) of the reference score. The user may change the dataset or the -tolerance. Ask whether they already have a reference score. If not, decide +tolerance. + +Ask where the dataset comes from: + +- **A local copy:** ask for its path. If it is missing or empty, tell the + user and ask again or switch to downloading; do not stop on it. On Slurm, + the path must be under a container mount: if it is not, propose adding its + directory to `container_mounts` and show the updated list. +- **Download:** the agent finds and downloads the dataset itself. + +Then ask about the reference score. Ask whether they already have a reference score. If not, decide whether HF can run the checkpoint (below), tell the user the verdict and why, and ask them to confirm it. @@ -165,9 +193,8 @@ Ask with one multi-select question: - **Disagg** gets its own criterion: `trtllm-eval` runs the model in-process and cannot drive a disaggregated server. - **Multimodal chosen:** add MMMU with the same rules as GSM8K (reference - score or measured or required baseline, same tolerance, same feature set). - Check now that `LLM_MODELS_ROOT/datasets/MMMU` exists and is non-empty, - and warn if not. + score or measured or required baseline, same tolerance, same feature set), + and ask where its data comes from as for GSM8K. **Multimodal model but not chosen:** add the text-only tip. ## Finish @@ -176,16 +203,28 @@ Ask with one multi-select question: 2. Ask for the workspace directory; write `/task.yaml`. If the directory does not exist, say so and create it once the user agrees. If the file exists, ask before overwriting. -3. Validate with agent-flow's own schema, fix, and re-run until it passes: - ```bash - python3 .claude/skills/modeling-task-yaml/scripts/validate_task_yaml.py /task.yaml - ``` -4. Give the run command: +3. Check it with the workflow's own check, take each problem back to the + user, fix, and re-run until it passes: + - when the user's agent-flow has `agent_flow/workflows/modeling_v2_onboard/`: + `modeling-v2-onboard --task /task.yaml --workspace --check`. + It checks the schema, the checkout, the workspace, the checkpoint, the + image, the mounts and the GPU pool. It does not create the workspace's + run state. + - otherwise: `python3 .claude/skills/modeling-task-yaml/scripts/validate_task_yaml.py /task.yaml` + (the schema only). + + If the command is not on PATH, install agent-flow first: + `pip install -e /agent-flow`. +4. **Slurm with modeling-v2-onboard only:** ask whether to run the runtime + check now. It submits one short Slurm job that starts the container and + shows the GPUs, the mounted paths and CUDA PyTorch, but it waits for a + node. Skipping it starts the run sooner: the run's environment task makes + the same checks once the planner has started. If the user wants it, add + `--smoke` to the check command and report its result. +5. Give the run command: `modeling-bringup --task /task.yaml --workspace `. - If `modeling-bringup` is not on PATH, add the install step - `pip install -e /agent-flow`. Mention - `modeling-v2-onboard` (same arguments) only when the user's agent-flow - has `agent_flow/workflows/modeling_v2_onboard/`. + Mention `modeling-v2-onboard` (same arguments) only when the user's + agent-flow has `agent_flow/workflows/modeling_v2_onboard/`. ## Sentence library @@ -203,7 +242,8 @@ Fill the `<...>` slots; keep the wording. `implements_tips`: - `"Target GPU: ()."` -- `"LLM_MODELS_ROOT=; GSM8K is under datasets/openai/gsm8k<; MMMU under datasets/MMMU>."` +- Local dataset: `" data: (pass it to trtllm-eval with --dataset_path)."` +- Downloaded dataset: `" data: no local copy; find and download it yourself, where the evaluation runs can reach it."` - Chat template: `"The checkpoint ships a chat template (); decide whether evaluation must apply it and use the same choice for the reference."` - Spec/disagg notes: `": ."` - Spec/disagg, no notes: `": the algorithm and configuration are left to the planner to infer from the checkpoint and the codebase."` @@ -220,10 +260,21 @@ trtllm_repo_path: /path/to/TensorRT-LLM slurm-environment: # only when the user runs on Slurm slurm_partition: docker_image: /path/to/image.sqsh + # without a GPU pool: account: qos: container_mounts: - /path/to/mount + # with a GPU pool, instead of the three above: + gpu_pool: + nodes: + max_nodes: + account: + qos: + gpus_per_node: + hours: + container_mounts: + - /path/to/mount completion_criteria: - "" @@ -233,7 +284,7 @@ completion_criteria: implements_tips: - "" - - "" + - "" - "" ``` @@ -246,5 +297,7 @@ implements_tips: | "Slurm config inferred and verified, please confirm" with no values shown | List every field with its value, source and check result | | Folding disagg into the `trtllm-eval` feature criterion | Disagg gets its own criterion | | "Agent measures the HF reference" for an NVFP4 / oversized checkpoint | Require a score or another baseline | -| Putting account / qos / mounts under `gpu_pool` | Directly under `slurm-environment` | -| Importing `agent_flow` to validate | Use `scripts/validate_task_yaml.py` | +| Account / qos / mounts both under `slurm-environment` and under `gpu_pool` | Under `gpu_pool` with a pool, directly under `slurm-environment` without | +| Asking the user for pool fields Slurm or the environment can answer | Ask only `nodes` and `max_nodes`; infer the rest and confirm it | +| Checking paths, image or mounts by hand during the conversation | Run the workflow's check at Finish | +| Running `--smoke` without asking | It waits for a Slurm node; ask first | diff --git a/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md b/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md index 01473e8515ba..03ab366d74bd 100644 --- a/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md +++ b/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md @@ -20,13 +20,14 @@ `agent-flow/agent_flow/workflows/modeling_bringup/task_schema.py`,该文件已在 main 上。 - 分四个部分对话:基础配置 → 运行方式 → 精度验证 → 功能验证。 - 把用户的选择展开成清晰、可验证的英文句子,写进 `completion_criteria` 和 `implements_tips`。 -- 写出文件后,用 `load_and_validate_task_yaml` 校验,并告诉用户怎样运行。 +- 写出文件后,用工作流自己的检查来校验(`modeling-v2-onboard --check`,没有 v2-onboard 时用 + `load_and_validate_task_yaml`),并告诉用户怎样运行。对话中不再重复实现这些检查。 ## 非目标 -- `modeling-v2-onboard` 特有的内容:`gpu_pool`(节点池)、并行拓扑选择、性能 Pareto、性能模型。 - 这些等 v2-onboard 合入 main 后再扩展。 -- 修改 agent-flow 的 schema 或工作流代码。 +- `modeling-v2-onboard` 特有的内容中,除 `gpu_pool`(节点池)以外的部分:并行拓扑选择、 + 性能 Pareto、性能模型。这些等 v2-onboard 合入 main 后再扩展。 +- 修改 agent-flow 的 schema 或工作流代码(`--check` 由 v2-onboard 分支提供)。 - 不问、也不写入环境细节:PATH/PYTHONPATH/HOME、HF 离线变量、MPI/UCX 变量、编译产物软链接。 用户主动提供时,可以原样写进 `implements_tips`。 @@ -53,17 +54,19 @@ 总的原则: - 第 1、2 部分各用一张表单,一条消息问完这一部分的所有字段;第 3、4 部分一次只问一件事。 - 有固定候选的问题,用选择题,即 AskUserQuestion。 -- 每个部分结束时,用表格逐字段列出具体值、来源和检查结果,请用户确认;不能只说"已核实"。 +- 每个部分结束时,用表格逐字段列出具体值和来源,请用户确认;不能只说"已核实"。 ### 第 1 部分:基础配置 -| 项 | 写入位置 | 检查 | +| 项 | 写入位置 | 说明 | |---|---|---| -| `reference_code_path`(HF modeling 文件) | 顶层 | 必须存在 | -| `checkpoint_path`(HF checkpoint 目录) | 顶层 | 必须存在,并且有 `config.json` | -| `trtllm_repo_path` | 顶层 | 必须存在,并且是 git checkout | -| 目标 GPU(例如 GB300 / sm_103) | `implements_tips` 的一句话 | 能从 partition 或本机推断时,先推断再确认 | -| `LLM_MODELS_ROOT` | `implements_tips` 的一句话 | 目录必须存在;检查下面有没有 `datasets/openai/gsm8k`,选了多模态时还要检查 MMMU 数据,没有就提醒用户 | +| `reference_code_path`(HF modeling 文件) | 顶层 | — | +| `checkpoint_path`(HF checkpoint 目录) | 顶层 | — | +| `trtllm_repo_path` | 顶层 | 干净的 checkout | +| 目标 GPU(例如 GB300 / sm_103) | `implements_tips` 的一句话 | 能从本机推断时,先推断再确认 | + +路径、镜像、mounts 等检查统一放在收尾时的 `--check` 里做。不再问 `LLM_MODELS_ROOT`: +运行只从 `checkpoint_path` 读权重,`LLM_MODELS_ROOT` 只用来找评测数据集,数据集改在第 3 部分直接问。 拿到 checkpoint 之后,读取 `config.json` 等文件,推断以下信息,供后面几部分使用: - 架构名; @@ -78,20 +81,18 @@ 先问"是否使用 slurm 模式"。 - 不用:不写 `slurm-environment`。 -- 用:问以下 5 项,全部写在 `slurm-environment` 下面,不写 `gpu_pool`。 - - `slurm_partition` - - `docker_image`:`.sqsh` 文件必须存在 - - `account` - - `qos` - - `container_mounts` - -`modeling-bringup` 的 prompt 会让 agent 读取整段 `slurm-environment`,所以 account、qos、 -`container_mounts` 写在这里,agent 能看到。 +- 用:问 `slurm_partition`、`docker_image`,以及(agent-flow 有 v2-onboard 时)是否使用 GPU pool。 + - 不用 pool:`account`、`qos`、`container_mounts` 直接写在 `slurm-environment` 下面,agent 会读到。 + - 用 pool:这些字段写在 `slurm-environment.gpu_pool` 下面,不在两处重复。用户只填 `nodes` 和 + `max_nodes`;`account`、`qos`、`gpus_per_node`、`hours`、`container_mounts` 由 skill 从当前环境 + (`sacctmgr`、`sinfo`)、旧的 task.yaml 或 memory 推断,列出来源请用户确认。 ### 第 3 部分:精度验证 **e2e 精度** - 默认用 GSM8K,通过 `trtllm-eval` 跑。标准是和 reference 分数相差小于 1%(绝对分差)。 +- 问数据集来源:用户自己给本地路径(缺失只提醒;Slurm 下要在某个 mount 下面),或者在线下载 + (让 agent 自己搜索并下载)。多模态的 MMMU 同样处理。 - 先问用户有没有 reference 分数: - 有:把具体数值写进标准。 - 没有:先判断 HF 实现能不能跑这个 checkpoint,依据是 `config.json` 里的 @@ -145,9 +146,12 @@ 1. 展示完整的 task.yaml 草稿,请用户确认或修改。 2. 问 workspace 路径,把 `task.yaml` 写进去。如果目标文件已经存在,先问用户,不能直接覆盖。 -3. 用 agent-flow 的 `load_and_validate_task_yaml` 校验。校验失败时,把错误告诉用户, - 改完再校验。 -4. 告诉用户运行命令,例如 `modeling-bringup --task --workspace `。 +3. 用 `modeling-v2-onboard --check` 校验(schema、checkout、workspace、checkpoint、镜像、mounts、 + GPU pool);没有 v2-onboard 时用 `load_and_validate_task_yaml`。失败时把问题告诉用户,改完再校验。 +4. Slurm + v2-onboard 时,问用户要不要现在跑运行时检查(`--check --smoke`:提交一个短 Slurm 作业, + 检查容器、GPU、mounts、CUDA PyTorch;要等节点)。不跑的话,planner 启动后由环境任务做同样的检查, + 并产出 worker 用的 wrapper。 +5. 告诉用户运行命令,例如 `modeling-bringup --task --workspace `。 ## 交给实现者决定的事 From 45d733921cea41f3d8ab63bd25e767283267a830 Mon Sep 17 00:00:00 2001 From: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:16:26 -0800 Subject: [PATCH 03/13] [None][feat] modeling-task-yaml: serve the skill to Codex from .agents/skills The skill now lives in .agents/skills/modeling-task-yaml, the repo-level skill directory Codex reads. .claude/skills/modeling-task-yaml is a symlink to it, so Claude Code loads the same files. The script paths in SKILL.md point at .agents/skills. Signed-off-by: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> --- {.claude => .agents}/skills/modeling-task-yaml/SKILL.md | 4 ++-- .../skills/modeling-task-yaml/scripts/inspect_checkpoint.py | 0 .../skills/modeling-task-yaml/scripts/validate_task_yaml.py | 0 .claude/skills/modeling-task-yaml | 1 + .../specs/2026-10-25-modeling-task-yaml-skill-design.md | 5 +++-- 5 files changed, 6 insertions(+), 4 deletions(-) rename {.claude => .agents}/skills/modeling-task-yaml/SKILL.md (99%) rename {.claude => .agents}/skills/modeling-task-yaml/scripts/inspect_checkpoint.py (100%) rename {.claude => .agents}/skills/modeling-task-yaml/scripts/validate_task_yaml.py (100%) create mode 120000 .claude/skills/modeling-task-yaml diff --git a/.claude/skills/modeling-task-yaml/SKILL.md b/.agents/skills/modeling-task-yaml/SKILL.md similarity index 99% rename from .claude/skills/modeling-task-yaml/SKILL.md rename to .agents/skills/modeling-task-yaml/SKILL.md index 674012c84b8a..a2500bc8e2ba 100644 --- a/.claude/skills/modeling-task-yaml/SKILL.md +++ b/.agents/skills/modeling-task-yaml/SKILL.md @@ -78,7 +78,7 @@ Items and checks: Inspect the checkpoint with: ```bash -python3 .claude/skills/modeling-task-yaml/scripts/inspect_checkpoint.py +python3 .agents/skills/modeling-task-yaml/scripts/inspect_checkpoint.py ``` Show what it found (architecture, dtype, multimodal, MTP layers, @@ -210,7 +210,7 @@ Ask with one multi-select question: It checks the schema, the checkout, the workspace, the checkpoint, the image, the mounts and the GPU pool. It does not create the workspace's run state. - - otherwise: `python3 .claude/skills/modeling-task-yaml/scripts/validate_task_yaml.py /task.yaml` + - otherwise: `python3 .agents/skills/modeling-task-yaml/scripts/validate_task_yaml.py /task.yaml` (the schema only). If the command is not on PATH, install agent-flow first: diff --git a/.claude/skills/modeling-task-yaml/scripts/inspect_checkpoint.py b/.agents/skills/modeling-task-yaml/scripts/inspect_checkpoint.py similarity index 100% rename from .claude/skills/modeling-task-yaml/scripts/inspect_checkpoint.py rename to .agents/skills/modeling-task-yaml/scripts/inspect_checkpoint.py diff --git a/.claude/skills/modeling-task-yaml/scripts/validate_task_yaml.py b/.agents/skills/modeling-task-yaml/scripts/validate_task_yaml.py similarity index 100% rename from .claude/skills/modeling-task-yaml/scripts/validate_task_yaml.py rename to .agents/skills/modeling-task-yaml/scripts/validate_task_yaml.py diff --git a/.claude/skills/modeling-task-yaml b/.claude/skills/modeling-task-yaml new file mode 120000 index 000000000000..5f331d420174 --- /dev/null +++ b/.claude/skills/modeling-task-yaml @@ -0,0 +1 @@ +../../.agents/skills/modeling-task-yaml \ No newline at end of file diff --git a/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md b/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md index 03ab366d74bd..168604995ac8 100644 --- a/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md +++ b/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md @@ -39,8 +39,9 @@ ## Skill 位置与形式 -- 位置:`.claude/skills/modeling-task-yaml/SKILL.md`。名字按 `.claude/README.md` 的 - `-` 规则取。(我的决定) +- 位置:`.agents/skills/modeling-task-yaml/SKILL.md`,Codex 从这里读取; + `.claude/skills/modeling-task-yaml` 是指向它的软链接,供 Claude Code 读取。名字按 + `.claude/README.md` 的 `-` 规则取。(我的决定) - `SKILL.md` 描述对话流程。可以附带一份带注释的模板,或者一份标准句子库,用来存放下文 要写入的英文句子。 - schema 校验直接调用 agent-flow 里现有的 `load_and_validate_task_yaml`,不要另写一份 From a292170cc929746b6d1061433e39a1a875b19c0d Mon Sep 17 00:00:00 2001 From: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:46:09 -0800 Subject: [PATCH 04/13] [None][feat] modeling-task-yaml: ask fixed-choice questions in any harness Fixed-choice questions use the harness's question tool when it has one (AskUserQuestion in Claude Code, request_user_input in Codex) and fall back to a numbered text format otherwise, including for multi-select questions a single-choice tool cannot ask. The text format marks the defaults and says how to reply. Signed-off-by: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> --- .agents/skills/modeling-task-yaml/SKILL.md | 33 +++++++++++++++++-- ...6-10-25-modeling-task-yaml-skill-design.md | 4 ++- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/.agents/skills/modeling-task-yaml/SKILL.md b/.agents/skills/modeling-task-yaml/SKILL.md index a2500bc8e2ba..7aea585f2f7f 100644 --- a/.agents/skills/modeling-task-yaml/SKILL.md +++ b/.agents/skills/modeling-task-yaml/SKILL.md @@ -19,9 +19,9 @@ the answers into clear, checkable sentences. - Talk in the user's language; write the YAML sentences in English. - Part 1 and Part 2 are each asked as one form (see below), one part per - message; Parts 3 and 4 go one question per message. Use AskUserQuestion - for anything with fixed choices (the feature checklist, yes/no); without - it, list the choices in plain text. + message; Parts 3 and 4 go one question per message. Ask anything with + fixed choices (the feature checklist, yes/no) as described in + "Questions with fixed choices". - Show values, not verdicts. Whenever you propose, infer, or recap values, list every field with its exact value (full paths, names, numbers) and, where you checked it, the check result. "All verified" or "already filled @@ -40,6 +40,33 @@ the answers into clear, checkable sentences. do not re-implement its checks in the conversation. A problem it reports goes back to the user. +## Questions with fixed choices + +Use your harness's question tool when it has one and it fits the question: +`AskUserQuestion` in Claude Code, `request_user_input` in Codex. A +single-choice tool cannot ask a multi-select question; ask that one in text. +Codex lists `request_user_input` in Default mode only when the user started +it with `--enable default_mode_request_user_input`; when the tool is not in +your tool list, ask in text. + +Otherwise ask in text, in this shape: + +```text +Which features should the accuracy run prove? Reply with numbers separated +by commas (e.g. 1,2,3), or "default" to keep the marked ones. + 1. CUDA graph (default) + 2. overlap scheduler (default) + 3. chunked prefill (default) + 4. KV cache reuse +``` + +- Number every choice and mark the defaults with `(default)`. +- Say how to answer: one number, or numbers separated by commas for a + multi-select, or "default". +- A yes/no question is numbered too (`1. yes`, `2. no`). +- Repeat the parsed answer back before using it; ask again if a number is + out of range. + ## Forms for Parts 1 and 2 Part 1 and Part 2 each get one form: a single message asking for every item diff --git a/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md b/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md index 168604995ac8..63b6adc3490c 100644 --- a/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md +++ b/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md @@ -54,7 +54,9 @@ 总的原则: - 第 1、2 部分各用一张表单,一条消息问完这一部分的所有字段;第 3、4 部分一次只问一件事。 -- 有固定候选的问题,用选择题,即 AskUserQuestion。 +- 有固定候选的问题,用当前工具自带的选择题工具(Claude Code 的 AskUserQuestion、Codex 的 + `request_user_input`)。没有这类工具,或者多选题而工具只支持单选时,用统一的文本格式: + 选项编号、默认项标 `(default)`、说明回复方式(单个编号、逗号分隔的多个编号,或 "default")。 - 每个部分结束时,用表格逐字段列出具体值和来源,请用户确认;不能只说"已核实"。 ### 第 1 部分:基础配置 From 566c2ffd0fc6106eccec8d515d69e9e838de8989 Mon Sep 17 00:00:00 2001 From: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> Date: Wed, 11 Nov 2026 01:59:26 -0800 Subject: [PATCH 05/13] [None][feat] modeling-task-yaml: pre-fill the run-mode form from the environment The run-mode form now proposes every Slurm field it can find instead of leaving it blank: Slurm itself when sinfo is on PATH, the default GPU partition from sinfo, the CI image from jenkins/current_image_tags.properties (the SBSA image for a Grace-based target GPU), and the account and QOS from the user's sacctmgr association. An earlier task.yaml or memory takes precedence. A field is left blank only when no source answers it, and the form then says what was looked at. Signed-off-by: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> --- .agents/skills/modeling-task-yaml/SKILL.md | 37 +++++++++++++------ ...6-10-25-modeling-task-yaml-skill-design.md | 6 ++- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/.agents/skills/modeling-task-yaml/SKILL.md b/.agents/skills/modeling-task-yaml/SKILL.md index 7aea585f2f7f..0583bd427cb3 100644 --- a/.agents/skills/modeling-task-yaml/SKILL.md +++ b/.agents/skills/modeling-task-yaml/SKILL.md @@ -73,12 +73,15 @@ Part 1 and Part 2 each get one form: a single message asking for every item of that part as a numbered list the user answers in one reply. Part 2's form is sent only after Part 1 is confirmed. -Before each form, look at what is at hand: the user's messages, -`nvidia-smi`, the current checkout, the checkpoint directory (e.g. a -`modeling_*.py` in it). Pre-fill each value you found, written out in full -with its source (e.g. "from your message", "from nvidia-smi", "found in the -checkpoint directory"); leave the rest blank. In Part 2's form, the Slurm -fields are answered only if the user answers yes to "use Slurm?". +Look before you ask. Before each form, try every source the part lists +(Part 1: the user's messages, `nvidia-smi`, the current checkout, the +checkpoint directory, e.g. a `modeling_*.py` in it; Part 2: the sources in +its table) and pre-fill each value you found, written out in full with its +source (e.g. "from your message", "from nvidia-smi", "from sacctmgr"). +The user's job is to confirm or correct proposals, not to fill in a blank +form. Leave a field blank only when no source answers it, and then say what +you looked at. In Part 2's form, the Slurm fields are answered only if the +user answers yes to "use Slurm?"; pre-fill them anyway. When the reply arrives (and, in Part 1, after the checkpoint inspection), send one recap: a table of every field, its value and its source (Part 1 @@ -117,12 +120,24 @@ these facts. Whether the run uses Slurm: - **No:** omit `slurm-environment` entirely. -- **Yes:** ask for `slurm_partition` and `docker_image` (both required, +- **Yes:** propose `slurm_partition` and `docker_image` (both required, under `slurm-environment`), and — only when the user's agent-flow has - `agent_flow/workflows/modeling_v2_onboard/` — whether the run uses a + `agent_flow/workflows/modeling_v2_onboard/` — ask whether the run uses a **GPU pool**: a set of whole nodes the run holds from its start and shares among its workers, instead of each worker submitting its own jobs. +Where each proposal comes from (an earlier task.yaml the user names, or +your memory of this user, takes precedence over the defaults below): + +| Field | Source | +|---|---| +| use Slurm? | propose yes when `sinfo` is on PATH | +| `slurm_partition` | `sinfo` partitions with GPUs; propose the default one (marked `*`) and list the others | +| `docker_image` | the CI image in `/jenkins/current_image_tags.properties`: `LLM_SBSA_DOCKER_IMAGE` for a Grace-based target GPU (GH200, GB200, GB300), `LLM_DOCKER_IMAGE` otherwise; a local `.sqsh` the user names replaces it | +| `account` | `sacctmgr show assoc user=$USER`; propose the only account, or list them when there are several | +| `qos` | the same association's default QOS; when it has none, list its QOS values or propose omitting the field | +| `container_mounts` | as in the tables below | + **Without a pool**, these optional fields also go directly under `slurm-environment`; the agents read them: @@ -135,9 +150,8 @@ Whether the run uses Slurm: **With a pool**, they go under `slurm-environment.gpu_pool` instead, never in both places. The user gives only `nodes` (holder jobs submitted at the start) and `max_nodes` (the most the planner may grow it to; omit for the -worker count). Fill in the rest yourself from this environment, Slurm -(`sacctmgr`, `sinfo`), an earlier task.yaml or your memory, and show each -with its source for confirmation: +worker count). Fill in the rest yourself from the sources above, and show +each with its source for confirmation: | Field | Value | |---|---| @@ -325,6 +339,7 @@ implements_tips: | Folding disagg into the `trtllm-eval` feature criterion | Disagg gets its own criterion | | "Agent measures the HF reference" for an NVFP4 / oversized checkpoint | Require a score or another baseline | | Account / qos / mounts both under `slurm-environment` and under `gpu_pool` | Under `gpu_pool` with a pool, directly under `slurm-environment` without | +| A form of blank fields ("slurm_partition: to fill in") | Pre-fill every field a source answers; leave blank only what none does, and say what you looked at | | Asking the user for pool fields Slurm or the environment can answer | Ask only `nodes` and `max_nodes`; infer the rest and confirm it | | Checking paths, image or mounts by hand during the conversation | Run the workflow's check at Finish | | Running `--smoke` without asking | It waits for a Slurm node; ask first | diff --git a/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md b/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md index 63b6adc3490c..ca1eda9a0c26 100644 --- a/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md +++ b/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md @@ -84,7 +84,11 @@ 先问"是否使用 slurm 模式"。 - 不用:不写 `slurm-environment`。 -- 用:问 `slurm_partition`、`docker_image`,以及(agent-flow 有 v2-onboard 时)是否使用 GPU pool。 +- 用:`slurm_partition`、`docker_image`、`account`、`qos` 都先由 skill 推断再请用户确认,不让用户从空白开始填: + 有 `sinfo` 就建议用 Slurm;分区取 `sinfo` 里的默认 GPU 分区;镜像取仓库 + `jenkins/current_image_tags.properties` 里的 CI 镜像(Grace 架构 GPU 用 SBSA 版); + `account`、`qos` 取 `sacctmgr` 的 association。用户给的旧 task.yaml 或 memory 优先。 + agent-flow 有 v2-onboard 时,再问是否使用 GPU pool。 - 不用 pool:`account`、`qos`、`container_mounts` 直接写在 `slurm-environment` 下面,agent 会读到。 - 用 pool:这些字段写在 `slurm-environment.gpu_pool` 下面,不在两处重复。用户只填 `nodes` 和 `max_nodes`;`account`、`qos`、`gpus_per_node`、`hours`、`container_mounts` 由 skill 从当前环境 From c4a319f87b0c15b6df726dc0aa8a47e51a6e0619 Mon Sep 17 00:00:00 2001 From: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:17:37 -0800 Subject: [PATCH 06/13] [None][feat] modeling-task-yaml: keep the conversation in the user's first language The conversation language is chosen from the user's first message (Chinese or English) and used for every later message, including forms, recaps and choice labels, until the user asks to switch. Tool output, the locale and the English examples in the skill do not change it. Field names, paths and commands stay as they are, and the YAML sentences stay in English. Signed-off-by: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> --- .agents/skills/modeling-task-yaml/SKILL.md | 9 ++++++++- .../specs/2026-10-25-modeling-task-yaml-skill-design.md | 3 +++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.agents/skills/modeling-task-yaml/SKILL.md b/.agents/skills/modeling-task-yaml/SKILL.md index 0583bd427cb3..957c9b85991c 100644 --- a/.agents/skills/modeling-task-yaml/SKILL.md +++ b/.agents/skills/modeling-task-yaml/SKILL.md @@ -17,7 +17,14 @@ the answers into clear, checkable sentences. ## Conversation contract -- Talk in the user's language; write the YAML sentences in English. +- Language: pick it from the user's first message in this conversation — + Chinese if it is written in Chinese, English otherwise — and use it for + every message after that: forms, recaps, questions, choice labels and + explanations. Switch only when the user asks you to. Tool output, the + system locale and the language of files you read do not change it. The + examples in this skill are in English; translate them. Field names, + paths, commands and values stay as they are, and the YAML sentences are + always written in English. - Part 1 and Part 2 are each asked as one form (see below), one part per message; Parts 3 and 4 go one question per message. Ask anything with fixed choices (the feature checklist, yes/no) as described in diff --git a/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md b/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md index ca1eda9a0c26..b0719cdf9d85 100644 --- a/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md +++ b/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md @@ -53,6 +53,9 @@ ## 对话流程 总的原则: +- 对话语言按用户在这次对话里的第一条消息决定:中文就用中文,否则用英语,之后所有消息(表单、 + 确认表、提问、选项)都用这种语言,用户要求才切换。字段名、路径、命令保持原样;写进 YAML 的 + 句子始终是英文。 - 第 1、2 部分各用一张表单,一条消息问完这一部分的所有字段;第 3、4 部分一次只问一件事。 - 有固定候选的问题,用当前工具自带的选择题工具(Claude Code 的 AskUserQuestion、Codex 的 `request_user_input`)。没有这类工具,或者多选题而工具只支持单选时,用统一的文本格式: From b067b6977c4305baae9a6bf1cfe17e59dcc9de76 Mon Sep 17 00:00:00 2001 From: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:32:08 -0800 Subject: [PATCH 07/13] [None][feat] modeling-task-yaml: propose the workspace path instead of asking At Finish the skill proposes a new workspace under /agent-flow/workspace/ (ignored by agent-flow's .gitignore), named after the checkpoint directory plus -v2-onboard or -bringup, with -2, -3, ... appended when the name is taken. The user confirms or replaces it; a replacement that already holds a task.yaml is not overwritten without asking. Signed-off-by: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> --- .agents/skills/modeling-task-yaml/SKILL.md | 13 ++++++++++--- .../2026-10-25-modeling-task-yaml-skill-design.md | 5 ++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.agents/skills/modeling-task-yaml/SKILL.md b/.agents/skills/modeling-task-yaml/SKILL.md index 957c9b85991c..4036370bb6f3 100644 --- a/.agents/skills/modeling-task-yaml/SKILL.md +++ b/.agents/skills/modeling-task-yaml/SKILL.md @@ -248,9 +248,16 @@ Ask with one multi-select question: ## Finish 1. Show the complete YAML and ask for changes. -2. Ask for the workspace directory; write `/task.yaml`. If the - directory does not exist, say so and create it once the user agrees. If - the file exists, ask before overwriting. +2. Propose a new workspace directory, as a full path, for the user to + confirm or replace: `/agent-flow/workspace/` + (agent-flow's `.gitignore` excludes `workspace/`). `` is the + checkpoint directory's name in lower case plus `-v2-onboard` when the + user's agent-flow has `agent_flow/workflows/modeling_v2_onboard/`, + `-bringup` otherwise; if that directory already exists, append `-2`, + `-3`, ... until it does not. Once the user confirms, create the + directory and write `/task.yaml`. If the user gives an + existing directory that already holds a task.yaml, ask before + overwriting. 3. Check it with the workflow's own check, take each problem back to the user, fix, and re-run until it passes: - when the user's agent-flow has `agent_flow/workflows/modeling_v2_onboard/`: diff --git a/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md b/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md index b0719cdf9d85..8fdf186820cd 100644 --- a/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md +++ b/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md @@ -155,7 +155,10 @@ ### 收尾 1. 展示完整的 task.yaml 草稿,请用户确认或修改。 -2. 问 workspace 路径,把 `task.yaml` 写进去。如果目标文件已经存在,先问用户,不能直接覆盖。 +2. 直接给出一个新的 workspace 完整路径请用户确认或替换:`/agent-flow/workspace/` + (agent-flow 的 `.gitignore` 忽略 `workspace/`),`` 是 checkpoint 目录名的小写加 + `-v2-onboard` 或 `-bringup`,目录已存在就加 `-2`、`-3`。确认后创建目录并写入 `task.yaml`; + 用户换成的目录里已有 task.yaml 时,先问再覆盖。 3. 用 `modeling-v2-onboard --check` 校验(schema、checkout、workspace、checkpoint、镜像、mounts、 GPU pool);没有 v2-onboard 时用 `load_and_validate_task_yaml`。失败时把问题告诉用户,改完再校验。 4. Slurm + v2-onboard 时,问用户要不要现在跑运行时检查(`--check --smoke`:提交一个短 Slurm 作业, From 9adcb217ae4e42cd720a08e08457bf72a7ffea6b Mon Sep 17 00:00:00 2001 From: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> Date: Wed, 11 Nov 2026 04:20:46 -0800 Subject: [PATCH 08/13] [None][feat] modeling-task-yaml: workspace outside the checkout, backend choice and launch at Finish The proposed workspace is now /workspace/ under the agent-flow the workflow command is installed from, or beside the checkout when that agent-flow is inside trtllm_repo_path: modeling-v2-onboard refuses a workspace inside the checkout, because its final task commits everything untracked there. At the last step the skill asks which backend a modeling-v2-onboard run uses (Claude Code or Codex, adding --backend codex) and whether to start the run now from the session, detached with setsid nohup and a launch.log in the workspace. Signed-off-by: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> --- .agents/skills/modeling-task-yaml/SKILL.md | 32 +++++++++++++++---- ...6-10-25-modeling-task-yaml-skill-design.md | 15 ++++++--- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/.agents/skills/modeling-task-yaml/SKILL.md b/.agents/skills/modeling-task-yaml/SKILL.md index 4036370bb6f3..a2b2a00a5cae 100644 --- a/.agents/skills/modeling-task-yaml/SKILL.md +++ b/.agents/skills/modeling-task-yaml/SKILL.md @@ -249,8 +249,15 @@ Ask with one multi-select question: 1. Show the complete YAML and ask for changes. 2. Propose a new workspace directory, as a full path, for the user to - confirm or replace: `/agent-flow/workspace/` - (agent-flow's `.gitignore` excludes `workspace/`). `` is the + confirm or replace: `/workspace/`, where + `` is the agent-flow directory the user's workflow command + is installed from (the parent of the `agent_flow` package that the + command's Python imports; `workspace/` is in agent-flow's `.gitignore`). + When that directory is inside `trtllm_repo_path`, or no agent-flow is + installed yet, use the directory beside the checkout, + `/`, and say why: + modeling-v2-onboard refuses a workspace inside the checkout, because its + final task commits everything untracked there. `` is the checkpoint directory's name in lower case plus `-v2-onboard` when the user's agent-flow has `agent_flow/workflows/modeling_v2_onboard/`, `-bringup` otherwise; if that directory already exists, append `-2`, @@ -276,10 +283,23 @@ Ask with one multi-select question: node. Skipping it starts the run sooner: the run's environment task makes the same checks once the planner has started. If the user wants it, add `--smoke` to the check command and report its result. -5. Give the run command: - `modeling-bringup --task /task.yaml --workspace `. - Mention `modeling-v2-onboard` (same arguments) only when the user's - agent-flow has `agent_flow/workflows/modeling_v2_onboard/`. +5. Run command. + - **modeling-v2-onboard** (when the user's agent-flow has + `agent_flow/workflows/modeling_v2_onboard/`): ask which backend the + run's planner and workers use, Claude Code (the default) or Codex, + then give the command: + `modeling-v2-onboard --task /task.yaml --workspace `, + with `--backend codex` added for Codex. + - **Otherwise:** give + `modeling-bringup --task /task.yaml --workspace `; + its roles' backends are set by the workflow, so there is nothing to + choose. + + Then ask whether to start the run now from this session. If yes, start + it detached, so it outlives this conversation: + `setsid nohup > /launch.log 2>&1 &`. Report its + PID and the log path, and say that rerunning the same command resumes + the run. If no, the user runs the command themselves. ## Sentence library diff --git a/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md b/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md index 8fdf186820cd..e29c81ce3860 100644 --- a/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md +++ b/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md @@ -155,16 +155,21 @@ ### 收尾 1. 展示完整的 task.yaml 草稿,请用户确认或修改。 -2. 直接给出一个新的 workspace 完整路径请用户确认或替换:`/agent-flow/workspace/` - (agent-flow 的 `.gitignore` 忽略 `workspace/`),`` 是 checkpoint 目录名的小写加 - `-v2-onboard` 或 `-bringup`,目录已存在就加 `-2`、`-3`。确认后创建目录并写入 `task.yaml`; - 用户换成的目录里已有 task.yaml 时,先问再覆盖。 +2. 直接给出一个新的 workspace 完整路径请用户确认或替换:`/workspace/`,`` + 是用户的工作流命令实际安装自的 agent-flow 目录(`workspace/` 在它的 `.gitignore` 里)。这个目录在 + `trtllm_repo_path` 里面、或者还没装 agent-flow 时,改用 checkout 旁边的 `/`, + 并说明原因:modeling-v2-onboard 拒绝放在 checkout 里的 workspace,因为最后一个任务会提交那里所有未跟踪的文件。 + `` 是 checkpoint 目录名的小写加 `-v2-onboard` 或 `-bringup`,目录已存在就加 `-2`、`-3`。 + 确认后创建目录并写入 `task.yaml`;用户换成的目录里已有 task.yaml 时,先问再覆盖。 3. 用 `modeling-v2-onboard --check` 校验(schema、checkout、workspace、checkpoint、镜像、mounts、 GPU pool);没有 v2-onboard 时用 `load_and_validate_task_yaml`。失败时把问题告诉用户,改完再校验。 4. Slurm + v2-onboard 时,问用户要不要现在跑运行时检查(`--check --smoke`:提交一个短 Slurm 作业, 检查容器、GPU、mounts、CUDA PyTorch;要等节点)。不跑的话,planner 启动后由环境任务做同样的检查, 并产出 worker 用的 wrapper。 -5. 告诉用户运行命令,例如 `modeling-bringup --task --workspace `。 +5. 给出运行命令。modeling-v2-onboard 先问 run 里的 planner 和 worker 用 Claude Code(默认)还是 Codex, + Codex 时命令加 `--backend codex`;modeling-bringup 没有统一的后端开关,直接给命令。然后问要不要由当前会话 + 代为启动:要的话用 `setsid nohup <命令> > /launch.log 2>&1 &` 后台启动,报告 PID 和日志路径, + 并说明重跑同一条命令会接着跑;不要就由用户自己运行。 ## 交给实现者决定的事 From dbe80659664dad054daa67c57521efe0ee04c0fe Mon Sep 17 00:00:00 2001 From: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> Date: Wed, 11 Nov 2026 06:34:42 -0800 Subject: [PATCH 09/13] [None][feat] modeling-task-yaml: ask only for the backend, not the model or effort Signed-off-by: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> --- .agents/skills/modeling-task-yaml/SKILL.md | 4 +++- .../specs/2026-10-25-modeling-task-yaml-skill-design.md | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.agents/skills/modeling-task-yaml/SKILL.md b/.agents/skills/modeling-task-yaml/SKILL.md index a2b2a00a5cae..5e72e5bd530c 100644 --- a/.agents/skills/modeling-task-yaml/SKILL.md +++ b/.agents/skills/modeling-task-yaml/SKILL.md @@ -289,7 +289,9 @@ Ask with one multi-select question: run's planner and workers use, Claude Code (the default) or Codex, then give the command: `modeling-v2-onboard --task /task.yaml --workspace `, - with `--backend codex` added for Codex. + with `--backend codex` added for Codex. The backend is the only + question: the model and reasoning effort are the workflow's defaults + for that backend, so ask about neither and add no `--model`. - **Otherwise:** give `modeling-bringup --task /task.yaml --workspace `; its roles' backends are set by the workflow, so there is nothing to diff --git a/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md b/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md index e29c81ce3860..704ed660c9b7 100644 --- a/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md +++ b/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md @@ -167,7 +167,8 @@ 检查容器、GPU、mounts、CUDA PyTorch;要等节点)。不跑的话,planner 启动后由环境任务做同样的检查, 并产出 worker 用的 wrapper。 5. 给出运行命令。modeling-v2-onboard 先问 run 里的 planner 和 worker 用 Claude Code(默认)还是 Codex, - Codex 时命令加 `--backend codex`;modeling-bringup 没有统一的后端开关,直接给命令。然后问要不要由当前会话 + Codex 时命令加 `--backend codex`。只问后端:模型和推理强度用工作流代码里该后端的默认值,不问也不加 + `--model`;modeling-bringup 没有统一的后端开关,直接给命令。然后问要不要由当前会话 代为启动:要的话用 `setsid nohup <命令> > /launch.log 2>&1 &` 后台启动,报告 PID 和日志路径, 并说明重跑同一条命令会接着跑;不要就由用户自己运行。 From 31cdbb1be9385a2a8bf67af7af51cad3ad4eb39a Mon Sep 17 00:00:00 2001 From: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> Date: Wed, 11 Nov 2026 07:22:44 -0800 Subject: [PATCH 10/13] [None][feat] modeling-task-yaml: stay on the task_schema workflows and install nothing unasked Signed-off-by: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> --- .agents/skills/modeling-task-yaml/SKILL.md | 32 +++++++++++++++++-- ...6-10-25-modeling-task-yaml-skill-design.md | 6 ++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/.agents/skills/modeling-task-yaml/SKILL.md b/.agents/skills/modeling-task-yaml/SKILL.md index 5e72e5bd530c..275fcbadcc88 100644 --- a/.agents/skills/modeling-task-yaml/SKILL.md +++ b/.agents/skills/modeling-task-yaml/SKILL.md @@ -15,6 +15,29 @@ environment — lives in sentences under `completion_criteria` and `implements_tips`. Your job is to ask for what the user left out and turn the answers into clear, checkable sentences. +## Which workflow + +This skill writes the format of +`agent_flow/workflows/modeling_bringup/task_schema.py`, for two workflows +only: `modeling-v2-onboard` when the user's agent-flow has +`agent_flow/workflows/modeling_v2_onboard/`, `modeling-bringup` otherwise. + +- `trtllm-modeling-v2` (`agent_flow/workflows/trtllm_modeling_v2/`) is a + different workflow with its own spec (`arch`, `members`, `environment` + sessions, accuracy anchors) and its own `intake`. Never convert the + task.yaml to it, run its intake, or offer its command, even when the + user says "v2" or "ModelingV2". +- When the user asks for modeling-v2-onboard and their agent-flow does not + have it, say so and ask whether to write the task for modeling-bringup + instead; do not substitute another workflow. +- Everything the run needs is decided by the run itself: it holds its GPU + nodes and measures the reference scores. Nothing has to be prepared + before launch beyond what the workflow's check reports. If you find + yourself listing other preconditions (sessions to start, scores to + record first), you are on the wrong workflow. +- Run options (backend, model) are command-line flags, never task.yaml + fields. + ## Conversation contract - Language: pick it from the user's first message in this conversation — @@ -275,8 +298,10 @@ Ask with one multi-select question: - otherwise: `python3 .agents/skills/modeling-task-yaml/scripts/validate_task_yaml.py /task.yaml` (the schema only). - If the command is not on PATH, install agent-flow first: - `pip install -e /agent-flow`. + If the command is not on PATH, do not install anything or create an + environment on your own: tell the user it is missing and ask where to + install it (`pip install -e ` into the environment they run + it from). Without it, run only the schema check above. 4. **Slurm with modeling-v2-onboard only:** ask whether to run the runtime check now. It submits one short Slurm job that starts the container and shows the GPUs, the mounted paths and CUDA PyTorch, but it waits for a @@ -379,3 +404,6 @@ implements_tips: | Asking the user for pool fields Slurm or the environment can answer | Ask only `nodes` and `max_nodes`; infer the rest and confirm it | | Checking paths, image or mounts by hand during the conversation | Run the workflow's check at Finish | | Running `--smoke` without asking | It waits for a Slurm node; ask first | +| Rewriting the task for `trtllm-modeling-v2` or running its intake | Write the `task_schema.py` format for modeling-v2-onboard or modeling-bringup only | +| Telling the user to start GPU sessions or record a reference score before launch | The run holds its nodes and measures the reference itself | +| Creating a venv or installing agent-flow unasked | Say the command is missing and ask where to install it | diff --git a/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md b/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md index 704ed660c9b7..ff7a7725a3ad 100644 --- a/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md +++ b/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md @@ -23,6 +23,12 @@ - 写出文件后,用工作流自己的检查来校验(`modeling-v2-onboard --check`,没有 v2-onboard 时用 `load_and_validate_task_yaml`),并告诉用户怎样运行。对话中不再重复实现这些检查。 +- 只写 `task_schema.py` 的格式,只对应 modeling-v2-onboard(agent-flow 里有它时)和 modeling-bringup。 + `trtllm-modeling-v2` 是另一个工作流(`arch`/`members`/`environment` sessions/accuracy anchor), + 不转换成它的格式、不跑它的 intake、不给它的命令。GPU 节点和 reference 分数都由 run 自己解决, + 启动前不需要额外准备;backend、model 是命令行参数,不写进 YAML。命令不在 PATH 上时不自行建环境安装, + 先问用户。 + ## 非目标 - `modeling-v2-onboard` 特有的内容中,除 `gpu_pool`(节点池)以外的部分:并行拓扑选择、 From b16a5238ec124ae62e86998b94f89d48272ec714 Mon Sep 17 00:00:00 2001 From: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> Date: Wed, 11 Nov 2026 07:29:20 -0800 Subject: [PATCH 11/13] [None][chore] modeling-task-yaml: keep the design notes out of the tree Signed-off-by: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> --- ...6-10-25-modeling-task-yaml-skill-design.md | 188 ------------------ 1 file changed, 188 deletions(-) delete mode 100644 docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md diff --git a/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md b/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md deleted file mode 100644 index ff7a7725a3ad..000000000000 --- a/docs/superpowers/specs/2026-10-25-modeling-task-yaml-skill-design.md +++ /dev/null @@ -1,188 +0,0 @@ -# 设计:交互式生成 agent-flow modeling task.yaml 的 skill - -## 目的与动机 - -在 agent-flow 中接入一个新模型时,用户需要手写 `task.yaml`。schema 本身很简单:三个路径、 -可选的 `slurm-environment`、两个自由文本列表 `completion_criteria` 和 `implements_tips`。 -真正决定一次运行成败的内容,比如精度标准、要验证哪些功能、环境信息,都只能靠用户用自然语言 -写进这两个列表。这些内容容易漏写,也容易写得模糊。 - -这个 skill 通过对话引导用户填写: -- 已知字段逐项确认; -- 用户没有主动提到、但很重要的字段,主动询问; -- 能从 checkpoint 推断出来的信息,先推断再请用户确认; - -最后产出一份能直接交给工作流运行的 `task.yaml`。 - -## 目标 - -- 覆盖 `modeling-bringup` 和 `modeling-v2-onboard` 共用的 task.yaml 部分。两者共用 - `agent-flow/agent_flow/workflows/modeling_bringup/task_schema.py`,该文件已在 main 上。 -- 分四个部分对话:基础配置 → 运行方式 → 精度验证 → 功能验证。 -- 把用户的选择展开成清晰、可验证的英文句子,写进 `completion_criteria` 和 `implements_tips`。 -- 写出文件后,用工作流自己的检查来校验(`modeling-v2-onboard --check`,没有 v2-onboard 时用 - `load_and_validate_task_yaml`),并告诉用户怎样运行。对话中不再重复实现这些检查。 - -- 只写 `task_schema.py` 的格式,只对应 modeling-v2-onboard(agent-flow 里有它时)和 modeling-bringup。 - `trtllm-modeling-v2` 是另一个工作流(`arch`/`members`/`environment` sessions/accuracy anchor), - 不转换成它的格式、不跑它的 intake、不给它的命令。GPU 节点和 reference 分数都由 run 自己解决, - 启动前不需要额外准备;backend、model 是命令行参数,不写进 YAML。命令不在 PATH 上时不自行建环境安装, - 先问用户。 - -## 非目标 - -- `modeling-v2-onboard` 特有的内容中,除 `gpu_pool`(节点池)以外的部分:并行拓扑选择、 - 性能 Pareto、性能模型。这些等 v2-onboard 合入 main 后再扩展。 -- 修改 agent-flow 的 schema 或工作流代码(`--check` 由 v2-onboard 分支提供)。 -- 不问、也不写入环境细节:PATH/PYTHONPATH/HOME、HF 离线变量、MPI/UCX 变量、编译产物软链接。 - 用户主动提供时,可以原样写进 `implements_tips`。 - -## 反目标(技术上能用,但仍算失败) - -- 悄悄填入用户没听说过的默认值。所有默认值都必须在对话里说出来。 -- 一次抛出一长串问题,让用户像填表一样回答。 -- 生成的 task.yaml 通不过 schema 校验,或者里面的路径不存在。 - -## Skill 位置与形式 - -- 位置:`.agents/skills/modeling-task-yaml/SKILL.md`,Codex 从这里读取; - `.claude/skills/modeling-task-yaml` 是指向它的软链接,供 Claude Code 读取。名字按 - `.claude/README.md` 的 `-` 规则取。(我的决定) -- `SKILL.md` 描述对话流程。可以附带一份带注释的模板,或者一份标准句子库,用来存放下文 - 要写入的英文句子。 -- schema 校验直接调用 agent-flow 里现有的 `load_and_validate_task_yaml`,不要另写一份 - schema。导入方式:直接按文件路径加载 `task_schema.py`,例如用 `importlib` 从文件加载。 - 原因是通过 `agent_flow` 包导入会连带导入 `claude_agent_sdk`,而这个包在部分机器上 - 不可用。(我的决定) -- 验证方式:拿一个真实 checkpoint 手动走一遍对话流程,不加自动化测试。(我的决定) - -## 对话流程 - -总的原则: -- 对话语言按用户在这次对话里的第一条消息决定:中文就用中文,否则用英语,之后所有消息(表单、 - 确认表、提问、选项)都用这种语言,用户要求才切换。字段名、路径、命令保持原样;写进 YAML 的 - 句子始终是英文。 -- 第 1、2 部分各用一张表单,一条消息问完这一部分的所有字段;第 3、4 部分一次只问一件事。 -- 有固定候选的问题,用当前工具自带的选择题工具(Claude Code 的 AskUserQuestion、Codex 的 - `request_user_input`)。没有这类工具,或者多选题而工具只支持单选时,用统一的文本格式: - 选项编号、默认项标 `(default)`、说明回复方式(单个编号、逗号分隔的多个编号,或 "default")。 -- 每个部分结束时,用表格逐字段列出具体值和来源,请用户确认;不能只说"已核实"。 - -### 第 1 部分:基础配置 - -| 项 | 写入位置 | 说明 | -|---|---|---| -| `reference_code_path`(HF modeling 文件) | 顶层 | — | -| `checkpoint_path`(HF checkpoint 目录) | 顶层 | — | -| `trtllm_repo_path` | 顶层 | 干净的 checkout | -| 目标 GPU(例如 GB300 / sm_103) | `implements_tips` 的一句话 | 能从本机推断时,先推断再确认 | - -路径、镜像、mounts 等检查统一放在收尾时的 `--check` 里做。不再问 `LLM_MODELS_ROOT`: -运行只从 `checkpoint_path` 读权重,`LLM_MODELS_ROOT` 只用来找评测数据集,数据集改在第 3 部分直接问。 - -拿到 checkpoint 之后,读取 `config.json` 等文件,推断以下信息,供后面几部分使用: -- 架构名; -- 是否多模态,依据包括 `vision_config`、`preprocessor_config.json` 等; -- 是否带 MTP 层,例如 `num_nextn_predict_layers`; -- 是否有 `chat_template.jinja`。 - -推断出来的结论要说给用户听。 - -### 第 2 部分:运行方式 - -先问"是否使用 slurm 模式"。 - -- 不用:不写 `slurm-environment`。 -- 用:`slurm_partition`、`docker_image`、`account`、`qos` 都先由 skill 推断再请用户确认,不让用户从空白开始填: - 有 `sinfo` 就建议用 Slurm;分区取 `sinfo` 里的默认 GPU 分区;镜像取仓库 - `jenkins/current_image_tags.properties` 里的 CI 镜像(Grace 架构 GPU 用 SBSA 版); - `account`、`qos` 取 `sacctmgr` 的 association。用户给的旧 task.yaml 或 memory 优先。 - agent-flow 有 v2-onboard 时,再问是否使用 GPU pool。 - - 不用 pool:`account`、`qos`、`container_mounts` 直接写在 `slurm-environment` 下面,agent 会读到。 - - 用 pool:这些字段写在 `slurm-environment.gpu_pool` 下面,不在两处重复。用户只填 `nodes` 和 - `max_nodes`;`account`、`qos`、`gpus_per_node`、`hours`、`container_mounts` 由 skill 从当前环境 - (`sacctmgr`、`sinfo`)、旧的 task.yaml 或 memory 推断,列出来源请用户确认。 - -### 第 3 部分:精度验证 - -**e2e 精度** -- 默认用 GSM8K,通过 `trtllm-eval` 跑。标准是和 reference 分数相差小于 1%(绝对分差)。 -- 问数据集来源:用户自己给本地路径(缺失只提醒;Slurm 下要在某个 mount 下面),或者在线下载 - (让 agent 自己搜索并下载)。多模态的 MMMU 同样处理。 -- 先问用户有没有 reference 分数: - - 有:把具体数值写进标准。 - - 没有:先判断 HF 实现能不能跑这个 checkpoint,依据是 `config.json` 里的 - `quantization_config`(例如 NVFP4 这类 TRT-LLM 专用格式 HF 跑不了)和权重大小 - (能否在目标节点上用 HF 单进程跑)。 - - 能跑:写成"agent 用 HF 实现(`reference_code_path`),在同样的评测设置下自己跑出 - reference 分数,再和它比较"。 - - 跑不了:把原因告诉用户,并要求用户提供以下任意一种:一个具体分数;或者另一个基准, - 比如同一模型的 BF16 checkpoint 路径,或者公开发布的分数。不能写一条达不到的标准。 -- 用户可以换数据集,也可以改容忍度。 -- 如果第 4 部分勾选了多模态,再加 MMMU,规则和 GSM8K 完全相同。 -- 如果存在 `chat_template.jinja`,在 `implements_tips` 里提醒:评测时要确认是否需要套用 - chat template。 - -**单 op 精度**:默认写入 `completion_criteria`,原文就用这句: - -> "Every op matches its reference within 1 ulp of the output dtype (relative to each row's -> max magnitude). A wider band needs a stated derivation. A faster replacement need not match -> the previous op bit for bit." - -### 第 4 部分:功能验证 - -用多选题,候选如下: - -| 功能 | 默认 | -|---|---| -| CUDA graph | 勾选 | -| overlap scheduler | 勾选 | -| chunked prefill | 勾选 | -| KV cache reuse | 不勾选 | -| speculative decoding | 不勾选 | -| disaggregated serving | 不勾选 | -| 多模态(视觉部分) | 不勾选;只有第 1 部分识别为多模态时才出现 | - -- **验证方式**:所有勾选的功能同时打开,在这个配置下跑第 3 部分的 e2e 精度验证 - (GSM8K,勾选多模态时还有 MMMU),并达到标准。写成一条 `completion_criteria`, - 其中列出全部勾选的功能。 -- **disagg 例外**:`trtllm-eval` 在进程内跑模型,不支持 disagg。所以 disagg 不放进上面那次 - 合并运行,单独写一条标准:"GSM8K(勾选多模态时还有 MMMU)通过 `trtllm-serve` 的 disagg - 服务来跑,标准和 e2e 精度相同"。服务怎么搭建交给 planner 推断。 -- **spec decoding / disagg**:勾选后,问一句"有没有要补充的说明",用户可以不填。 - - 填了:把说明写进 `implements_tips`。 - - 没填:写一句 "left to the planner to infer from the checkpoint and the codebase"。 - - checkpoint 带 MTP 层时,可以把这个事实告诉用户,作为参考。 -- **多模态**: - - 勾选:精度标准加上 MMMU。 - - 识别为多模态但没有勾选:在 `implements_tips` 里写明 - "text model only; vision tower out of scope"。 - -### 收尾 - -1. 展示完整的 task.yaml 草稿,请用户确认或修改。 -2. 直接给出一个新的 workspace 完整路径请用户确认或替换:`/workspace/`,`` - 是用户的工作流命令实际安装自的 agent-flow 目录(`workspace/` 在它的 `.gitignore` 里)。这个目录在 - `trtllm_repo_path` 里面、或者还没装 agent-flow 时,改用 checkout 旁边的 `/`, - 并说明原因:modeling-v2-onboard 拒绝放在 checkout 里的 workspace,因为最后一个任务会提交那里所有未跟踪的文件。 - `` 是 checkpoint 目录名的小写加 `-v2-onboard` 或 `-bringup`,目录已存在就加 `-2`、`-3`。 - 确认后创建目录并写入 `task.yaml`;用户换成的目录里已有 task.yaml 时,先问再覆盖。 -3. 用 `modeling-v2-onboard --check` 校验(schema、checkout、workspace、checkpoint、镜像、mounts、 - GPU pool);没有 v2-onboard 时用 `load_and_validate_task_yaml`。失败时把问题告诉用户,改完再校验。 -4. Slurm + v2-onboard 时,问用户要不要现在跑运行时检查(`--check --smoke`:提交一个短 Slurm 作业, - 检查容器、GPU、mounts、CUDA PyTorch;要等节点)。不跑的话,planner 启动后由环境任务做同样的检查, - 并产出 worker 用的 wrapper。 -5. 给出运行命令。modeling-v2-onboard 先问 run 里的 planner 和 worker 用 Claude Code(默认)还是 Codex, - Codex 时命令加 `--backend codex`。只问后端:模型和推理强度用工作流代码里该后端的默认值,不问也不加 - `--model`;modeling-bringup 没有统一的后端开关,直接给命令。然后问要不要由当前会话 - 代为启动:要的话用 `setsid nohup <命令> > /launch.log 2>&1 &` 后台启动,报告 PID 和日志路径, - 并说明重跑同一条命令会接着跑;不要就由用户自己运行。 - -## 交给实现者决定的事 - -- skill 的具体名字和 frontmatter 里的 description 怎么写。description 要让 Claude 在 - "写 / 生成 modeling task.yaml、接入新模型到 agent-flow" 这类请求下触发。 -- 标准句子的最终英文措辞。除单 op 精度那句以外,其余句子保持精简、可以验证即可。 -- 用模板文件还是在 SKILL.md 里写示例。 -- 怎样找到 agent-flow 里的 `task_schema.py`:优先用 `trtllm_repo_path` 下的 - `agent-flow/`,找不到时再问用户。 From aa122b6cf191bcb2fd1b3cadfa26ec77208c3c74 Mon Sep 17 00:00:00 2001 From: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:57:28 -0800 Subject: [PATCH 12/13] [None][feat] modeling-v2-onboard-task: rename the skill, read Qwen3.5 MTP keys, drop --check/--smoke - Rename modeling-task-yaml to modeling-v2-onboard-task. - inspect_checkpoint.py reads the MTP layer count from the keys in trtllm_modeling_v2's DRAFT_LAYER_KEYS, so Qwen3.5-style checkpoints (text_config.mtp_num_hidden_layers) report it. - modeling-v2-onboard has no --check or --smoke; Finish now runs the schema check for both workflows and leaves the image, mounts and GPU pool to the run. Signed-off-by: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> --- .../SKILL.md | 47 ++++++++----------- .../scripts/inspect_checkpoint.py | 10 +++- .../scripts/validate_task_yaml.py | 0 .claude/skills/modeling-task-yaml | 1 - .claude/skills/modeling-v2-onboard-task | 1 + 5 files changed, 29 insertions(+), 30 deletions(-) rename .agents/skills/{modeling-task-yaml => modeling-v2-onboard-task}/SKILL.md (92%) rename .agents/skills/{modeling-task-yaml => modeling-v2-onboard-task}/scripts/inspect_checkpoint.py (94%) rename .agents/skills/{modeling-task-yaml => modeling-v2-onboard-task}/scripts/validate_task_yaml.py (100%) delete mode 120000 .claude/skills/modeling-task-yaml create mode 120000 .claude/skills/modeling-v2-onboard-task diff --git a/.agents/skills/modeling-task-yaml/SKILL.md b/.agents/skills/modeling-v2-onboard-task/SKILL.md similarity index 92% rename from .agents/skills/modeling-task-yaml/SKILL.md rename to .agents/skills/modeling-v2-onboard-task/SKILL.md index 275fcbadcc88..47b87150996e 100644 --- a/.agents/skills/modeling-task-yaml/SKILL.md +++ b/.agents/skills/modeling-v2-onboard-task/SKILL.md @@ -1,5 +1,5 @@ --- -name: modeling-task-yaml +name: modeling-v2-onboard-task description: Use when a user wants to write, generate, or fill in a task.yaml (task brief) for an agent-flow modeling workflow (modeling-bringup or modeling-v2-onboard), or asks how to onboard / bring up a new model with agent-flow. license: Apache-2.0 metadata: @@ -32,7 +32,7 @@ only: `modeling-v2-onboard` when the user's agent-flow has instead; do not substitute another workflow. - Everything the run needs is decided by the run itself: it holds its GPU nodes and measures the reference scores. Nothing has to be prepared - before launch beyond what the workflow's check reports. If you find + before launch beyond a task.yaml that passes the schema check. If you find yourself listing other preconditions (sessions to start, scores to record first), you are on the wrong workflow. - Run options (backend, model) are command-line flags, never task.yaml @@ -66,8 +66,9 @@ only: `modeling-v2-onboard` when the user's agent-flow has (PATH, PYTHONPATH, HOME, HF offline, MPI/UCX) or build-artifact symlinks. If the user volunteers such details, copy them verbatim into `implements_tips`. -- Check paths and settings once, at Finish, with the workflow's own check; - do not re-implement its checks in the conversation. A problem it reports +- Do not check paths, the image or the mounts by hand during the + conversation: the run's environment task checks them once the planner + has started. At Finish, run only the schema check; a problem it reports goes back to the user. ## Questions with fixed choices @@ -138,7 +139,7 @@ Items and checks: Inspect the checkpoint with: ```bash -python3 .agents/skills/modeling-task-yaml/scripts/inspect_checkpoint.py +python3 .agents/skills/modeling-v2-onboard-task/scripts/inspect_checkpoint.py ``` Show what it found (architecture, dtype, multimodal, MTP layers, @@ -288,27 +289,13 @@ Ask with one multi-select question: directory and write `/task.yaml`. If the user gives an existing directory that already holds a task.yaml, ask before overwriting. -3. Check it with the workflow's own check, take each problem back to the - user, fix, and re-run until it passes: - - when the user's agent-flow has `agent_flow/workflows/modeling_v2_onboard/`: - `modeling-v2-onboard --task /task.yaml --workspace --check`. - It checks the schema, the checkout, the workspace, the checkpoint, the - image, the mounts and the GPU pool. It does not create the workspace's - run state. - - otherwise: `python3 .agents/skills/modeling-task-yaml/scripts/validate_task_yaml.py /task.yaml` - (the schema only). - - If the command is not on PATH, do not install anything or create an - environment on your own: tell the user it is missing and ask where to - install it (`pip install -e ` into the environment they run - it from). Without it, run only the schema check above. -4. **Slurm with modeling-v2-onboard only:** ask whether to run the runtime - check now. It submits one short Slurm job that starts the container and - shows the GPUs, the mounted paths and CUDA PyTorch, but it waits for a - node. Skipping it starts the run sooner: the run's environment task makes - the same checks once the planner has started. If the user wants it, add - `--smoke` to the check command and report its result. -5. Run command. +3. Check it against agent-flow's task schema, which both workflows load, + take each problem back to the user, fix, and re-run until it passes: + `python3 .agents/skills/modeling-v2-onboard-task/scripts/validate_task_yaml.py /task.yaml`. + It checks the fields and that the three paths exist; the image, the + mounts and the GPU pool are checked by the run once the planner has + started. +4. Run command. - **modeling-v2-onboard** (when the user's agent-flow has `agent_flow/workflows/modeling_v2_onboard/`): ask which backend the run's planner and workers use, Claude Code (the default) or Codex, @@ -322,6 +309,11 @@ Ask with one multi-select question: its roles' backends are set by the workflow, so there is nothing to choose. + If the command is not on PATH, do not install anything or create an + environment on your own: tell the user it is missing and ask where to + install it (`pip install -e ` into the environment they run + it from). + Then ask whether to start the run now from this session. If yes, start it detached, so it outlives this conversation: `setsid nohup > /launch.log 2>&1 &`. Report its @@ -402,8 +394,7 @@ implements_tips: | Account / qos / mounts both under `slurm-environment` and under `gpu_pool` | Under `gpu_pool` with a pool, directly under `slurm-environment` without | | A form of blank fields ("slurm_partition: to fill in") | Pre-fill every field a source answers; leave blank only what none does, and say what you looked at | | Asking the user for pool fields Slurm or the environment can answer | Ask only `nodes` and `max_nodes`; infer the rest and confirm it | -| Checking paths, image or mounts by hand during the conversation | Run the workflow's check at Finish | -| Running `--smoke` without asking | It waits for a Slurm node; ask first | +| Checking paths, image or mounts by hand during the conversation | Run the schema check at Finish; the run checks the rest | | Rewriting the task for `trtllm-modeling-v2` or running its intake | Write the `task_schema.py` format for modeling-v2-onboard or modeling-bringup only | | Telling the user to start GPU sessions or record a reference score before launch | The run holds its nodes and measures the reference itself | | Creating a venv or installing agent-flow unasked | Say the command is missing and ask where to install it | diff --git a/.agents/skills/modeling-task-yaml/scripts/inspect_checkpoint.py b/.agents/skills/modeling-v2-onboard-task/scripts/inspect_checkpoint.py similarity index 94% rename from .agents/skills/modeling-task-yaml/scripts/inspect_checkpoint.py rename to .agents/skills/modeling-v2-onboard-task/scripts/inspect_checkpoint.py index 4bb0b8bc0002..34b4e686d724 100644 --- a/.agents/skills/modeling-task-yaml/scripts/inspect_checkpoint.py +++ b/.agents/skills/modeling-v2-onboard-task/scripts/inspect_checkpoint.py @@ -32,7 +32,15 @@ MULTIMODAL_CONFIG_KEYS = ("vision_config", "audio_config", "visual", "vision_tower") PROCESSOR_FILES = ("preprocessor_config.json", "processor_config.json") -MTP_KEYS = ("num_nextn_predict_layers", "num_mtp_layers", "mtp_num_layers") +# Where a config states its MTP layer count; the first three are +# DRAFT_LAYER_KEYS in trtllm_modeling_v2's spec/derive.py. +MTP_KEYS = ( + "num_nextn_predict_layers", + "num_mtp_modules", + "mtp_num_hidden_layers", + "num_mtp_layers", + "mtp_num_layers", +) WEIGHT_SUFFIXES = (".safetensors", ".bin", ".pt") diff --git a/.agents/skills/modeling-task-yaml/scripts/validate_task_yaml.py b/.agents/skills/modeling-v2-onboard-task/scripts/validate_task_yaml.py similarity index 100% rename from .agents/skills/modeling-task-yaml/scripts/validate_task_yaml.py rename to .agents/skills/modeling-v2-onboard-task/scripts/validate_task_yaml.py diff --git a/.claude/skills/modeling-task-yaml b/.claude/skills/modeling-task-yaml deleted file mode 120000 index 5f331d420174..000000000000 --- a/.claude/skills/modeling-task-yaml +++ /dev/null @@ -1 +0,0 @@ -../../.agents/skills/modeling-task-yaml \ No newline at end of file diff --git a/.claude/skills/modeling-v2-onboard-task b/.claude/skills/modeling-v2-onboard-task new file mode 120000 index 000000000000..b3c80da026fb --- /dev/null +++ b/.claude/skills/modeling-v2-onboard-task @@ -0,0 +1 @@ +../../.agents/skills/modeling-v2-onboard-task \ No newline at end of file From f818e5c19f09a0c8bfba59f3f470b9098ae0606f Mon Sep 17 00:00:00 2001 From: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:51:02 -0800 Subject: [PATCH 13/13] [None][fix] modeling-v2-onboard-task: never load the schema from a path the task names validate_task_yaml.py executes task_schema.py, so it took the schema from /agent-flow, a path the YAML under check chose. It now uses --agent-flow when given, otherwise the agent-flow this Python would import (located with find_spec, which does not execute the package), then this checkout's agent-flow. Signed-off-by: Fred Wei <20514172+WeiHaocheng@users.noreply.github.com> --- .../scripts/validate_task_yaml.py | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/.agents/skills/modeling-v2-onboard-task/scripts/validate_task_yaml.py b/.agents/skills/modeling-v2-onboard-task/scripts/validate_task_yaml.py index 57aeaebba6f6..3919f03a0137 100644 --- a/.agents/skills/modeling-v2-onboard-task/scripts/validate_task_yaml.py +++ b/.agents/skills/modeling-v2-onboard-task/scripts/validate_task_yaml.py @@ -23,8 +23,9 @@ Usage: python3 validate_task_yaml.py /path/to/task.yaml [--agent-flow DIR] -Without --agent-flow, the schema is looked up under the task's -trtllm_repo_path, then under this checkout. +Without --agent-flow, the schema comes from the agent-flow that this +Python would import, then from this checkout. Never from a path the task +names: the schema file is executed, so it must not be chosen by the YAML. """ import argparse @@ -32,23 +33,18 @@ import sys from pathlib import Path -import yaml - SCHEMA_RELPATH = Path("agent_flow/workflows/modeling_bringup/task_schema.py") THIS_CHECKOUT = Path(__file__).resolve().parents[4] -def _candidate_agent_flow_dirs(task_path: Path, explicit: Path | None) -> list[Path]: +def _candidate_agent_flow_dirs(explicit: Path | None) -> list[Path]: if explicit is not None: return [explicit] candidates = [] - try: - data = yaml.safe_load(task_path.read_text(encoding="utf-8")) or {} - except (OSError, yaml.YAMLError): - data = {} - repo = data.get("trtllm_repo_path") if isinstance(data, dict) else None - if isinstance(repo, str) and repo.strip(): - candidates.append(Path(repo) / "agent-flow") + # find_spec locates a top-level package without executing it. + installed = importlib.util.find_spec("agent_flow") + if installed is not None and installed.origin: + candidates.append(Path(installed.origin).resolve().parents[1]) candidates.append(THIS_CHECKOUT / "agent-flow") return candidates @@ -67,7 +63,7 @@ def main() -> int: parser.add_argument("--agent-flow", type=Path, help="agent-flow directory holding the schema") args = parser.parse_args() - candidates = _candidate_agent_flow_dirs(args.task, args.agent_flow) + candidates = _candidate_agent_flow_dirs(args.agent_flow) agent_flow_dir = next((d for d in candidates if (d / SCHEMA_RELPATH).is_file()), None) if agent_flow_dir is None: tried = ", ".join(str(d / SCHEMA_RELPATH) for d in candidates)