diff --git a/.dev.vars.example b/.dev.vars.example index 10733370..172baee8 100644 --- a/.dev.vars.example +++ b/.dev.vars.example @@ -40,6 +40,17 @@ SEED_MAX_METADATA_SIZE=10485760 # 兼容旧变量名 ALLOW_SEED（新变量优先）。 SEED_SOURCE_ALLOWED_HOSTS= +# ── 任务系统 ──────────────────────────────────────────────────────────────── +# URL 离线下载的主机白名单，逗号/空格/分号分隔，支持 *.example.com。 +# **默认留空 = 拒绝一切外网下载**（防止被当作免费代理刷流量）。 +# 任务在执行时会重新校验该白名单，排队期间收紧配置也会生效。 +ALLOW_OFFLINE_DOWNLOAD= + +# 部署 origin（如 https://your-domain.com）。后台任务没有请求对象，存储驱动 +# 返回的相对下载直链（/d/...）需要它才能解析。Cloudflare 部署通常不需要填 +# （入队时会把当前请求的 origin 写进任务）；EdgeOne / Node 建议显式配置。 +PUBLIC_ORIGIN= + # 说明：seed_single_direct_preview 不是环境变量，而是数据库中的站点设置项 # （默认 false）。可在管理后台「设置 → 种子」中修改，单文件种子解析时生效。 diff --git a/.env.example b/.env.example index 79b6082e..b90f6624 100644 --- a/.env.example +++ b/.env.example @@ -86,3 +86,11 @@ CF_API_KEY= # 目录列表与下载直链每次都直接回源驱动，既不做跨请求缓存，也不合并并发相同 # 调用。相关环境变量（CACHE_* / SINGLEFLIGHT*）已全部删除，残留同名变量不会 # 再被读取。 + +# ── 任务系统 ──────────────────────────────────────────────────────────────── +# URL 离线下载的主机白名单，逗号/空格/分号分隔，支持 *.example.com。 +# 默认留空 = 拒绝一切外网下载。 +ALLOW_OFFLINE_DOWNLOAD= + +# 部署 origin（后台任务解析相对下载直链时需要）。 +PUBLIC_ORIGIN= diff --git a/README.md b/README.md index bca3fac3..5d83ce82 100644 --- a/README.md +++ b/README.md @@ -153,10 +153,50 @@ pnpm run deploy:worker | `DB_DRIVER` | `auto` | 可选 `auto` / `blob` / `cfkv` / `kv` / `d1` / `do` / `mysql` | | `DB_CIPHER` | `none` | 敏感字段落盘算法，如 `aes-256-gcm` | +> **KV 配额提示**：`DB_FORMAT=key` 会维护全局索引键（`openlist_key_index`）， +> 首次读取旧数据后 load/save 不再消耗 KV `list` 配额。但每次保存仍按记录数写入 +> （默认配置约 100+ 条），Cloudflare KV 免费版只有 1000 写/天，长期使用建议 +> `DB_DRIVER=d1` 或 `DB_FORMAT=map`。 + 完整环境变量，请参阅[配置变量](https://doc.oplist.org/ecosystem/official_worker/guide_env) +### 任务系统（异步操作） + +大文件复制/移动不再阻塞请求：超过阈值（默认 32MB）的操作会转成持久化任务， +由 `TaskRunner` Durable Object 用 alarm 自驱执行，进度可在前端任务页查看。 + +- **调度**：Cloudflare 上由 `TASK_DO`（TaskRunner）驱动，入队即开始执行；每分钟的 + cron 只是看门狗。没有 DO 绑定时（EdgeOne / ESA / Vercel / Node）自动降级为 + 「任务文档 + 请求内时间片」。 +- **单次 tick 预算**：设置项 `task_tick_budget_ms`，默认 `0`（自动）。Cloudflare + 自动值为 2000ms —— **免费计划的保守值**；付费计划建议改为 20000， + 必要时可调到 60000。 +- **并发**：`task_parallel_units` 默认 1，最大 4（Cloudflare 每次调用最多 6 条 + 出站连接，每个传输通常占 2 条）。 +- **免费计划提醒**：Workers 免费计划 CPU 上限很低，大文件流式传输容易触发 + CPU 超限；建议对大文件使用支持服务端直传（`putUrl`）的目标存储，或升级付费计划。 +- **非 Cloudflare 平台**：EdgeOne 的 `schedules` 最小间隔为一天，无法作为分钟级 + 任务泵。请用外部 cron 每分钟请求一次： + + ```bash + curl -X POST https://your-domain/api/task/run \ + -H "X-Cron-Secret: $JWT_SECRET" + ``` + + 该接口只执行已入队的任务，不接受任何路径/URL 参数。 + +任务相关设置（设置页「流量」分组）：`task_enabled`、`task_sync_threshold_mb`、 +`task_probe_max_entries`、`task_tick_budget_ms`、`task_parallel_units`、 +`task_keep_done` 以及各类型的 `task_*_max_retry`。 + +**URL 离线下载**也走同一套任务队列（`POST /api/fs/add_offline_download`）。 +出于安全考虑默认关闭：必须配置 `ALLOW_OFFLINE_DOWNLOAD`（主机白名单， +逗号分隔，支持 `*.example.com`），否则请求直接返回 403。目标存储没有流式/ +分片上传能力时，单文件受内存上限约束（可用 `TASK_DOWNLOAD_MAX_BYTES` 调整， +默认 64MB）。 + ## 帮助支持 diff --git a/readmes/README_en.md b/readmes/README_en.md index f013e124..30121000 100644 --- a/readmes/README_en.md +++ b/readmes/README_en.md @@ -152,6 +152,12 @@ After deployment, set the following variables: | `DB_DRIVER` | `auto` | `auto` / `blob` / `cfkv` / `kv` / `d1` / `do` / `mysql` | | `DB_CIPHER` | `none` | At-rest cipher for sensitive fields, e.g. `aes-256-gcm` | +> **KV quota note**: `DB_FORMAT=key` maintains a global index key (`openlist_key_index`), +> so after the first read of legacy data, load/save no longer consume KV `list` quota. +> Each save still writes one entry per record (about 100+ with the default config), and +> the Cloudflare KV free tier allows only 1000 writes/day — prefer `DB_DRIVER=d1` or +> `DB_FORMAT=map` for long-term use. + For the full variable list, see [Configuration variables](https://doc.oplist.org/ecosystem/official_worker/guide_env). ## Support diff --git a/scripts/test-task-api.mts b/scripts/test-task-api.mts index 2bd48ab1..d8d784d8 100644 --- a/scripts/test-task-api.mts +++ b/scripts/test-task-api.mts @@ -1,5 +1,23 @@ -// Quick smoke test for task management API endpoints -import app from "../src/backend/index" +// Smoke test for the task management API. +// +// 契约对齐 Go 版（server/handles/task.go）： +// - 列表/清理：200 +// - 单个任务操作（info/cancel/delete/retry）：tid 不存在 → 404（防探测） +// - 批量操作：200，返回 {tid: error} 映射 +import { Hono } from "hono" +import { saveDb } from "../src/backend/internal/model/db" +import { setupRouter } from "../src/backend/server/router" + +// 不用 src/backend/index 的默认导出：它带「存储配置错误 → 全局 503」拦截， +// 在无绑定的本地冒烟环境里会把所有 API 直接拦掉。这里只挂路由。 +const api = new Hono() +setupRouter(api) +const app = new Hono() +app.route("/api", api) + +const TOKEN = "TASK_API_SMOKE_TOKEN" +const env: any = { JWT_SECRET: "TASK_API_SMOKE_CRON_SECRET" } +const headers = { Authorization: `Bearer ${TOKEN}` } async function test(name: string, fn: () => Promise) { try { @@ -12,101 +30,107 @@ async function test(name: string, fn: () => Promise) { } async function req(method: string, path: string, body?: any) { - const res = await app.request(path, { - method, - headers: body !== undefined ? { "Content-Type": "application/json" } : {}, - body: body !== undefined ? JSON.stringify(body) : undefined, - }) + const res = await app.request( + path, + { + method, + headers: + body !== undefined + ? { ...headers, "Content-Type": "application/json" } + : headers, + body: body !== undefined ? JSON.stringify(body) : undefined, + }, + env, + ) const json = await res.json().catch(() => null) return { status: res.status, json } } -const types = ["upload", "copy", "move", "offline_download"] - -await test("GET /api/task/upload/undone returns task list", async () => { - const { status, json } = await req("GET", "/api/task/upload/undone") - if (status !== 200) throw new Error(`status ${status}`) - if (json.code !== 200) throw new Error(`code ${json.code}`) - if (!Array.isArray(json.data)) throw new Error("data is not an array") -}) - -await test("GET /api/task/upload/done returns task list", async () => { - const { status, json } = await req("GET", "/api/task/upload/done") - if (status !== 200) throw new Error(`status ${status}`) - if (json.code !== 200) throw new Error(`code ${json.code}`) - if (!Array.isArray(json.data)) throw new Error("data is not an array") -}) - -await test("GET /api/task/{all types}/{done} return 200", async () => { +await saveDb( + { + settings: [{ key: "token", value: TOKEN }], + users: [ + { + id: 1, + username: "admin", + password: "xxx", + role: 2, + permission: 0, + base_path: "/", + disabled: false, + }, + ], + storages: [], + shares: [], + }, + env, +) + +const types = [ + "upload", + "copy", + "move", + "offline_download", + "decompress", + "seed_generate", + "seed_save", +] + +await test("GET /api/task/{all types}/{undone|done} returns TaskInfo[]", async () => { for (const type of types) { - for (const done of ["undone", "done"]) { - const { status } = await req("GET", `/api/task/${type}/${done}`) - if (status !== 200) throw new Error(`${type}/${done}: status ${status}`) + for (const state of ["undone", "done"]) { + const { status, json } = await req("GET", `/api/task/${type}/${state}`) + if (status !== 200) throw new Error(`${type}/${state}: status ${status}`) + if (json.code !== 200) throw new Error(`${type}/${state}: code ${json.code}`) + if (!Array.isArray(json.data)) { + throw new Error(`${type}/${state}: data is not an array`) + } } } }) -await test("POST /api/task/upload/clear_done works", async () => { - const { status, json } = await req("POST", "/api/task/upload/clear_done") - if (status !== 200) throw new Error(`status ${status}`) - if (json.code !== 200) throw new Error(`code ${json.code}`) +await test("unknown task type returns 404", async () => { + const { status } = await req("GET", "/api/task/not-a-type/undone") + if (status !== 404) throw new Error(`status ${status}`) }) -await test("POST /api/task/upload/clear_succeeded works", async () => { - const { status, json } = await req("POST", "/api/task/upload/clear_succeeded") - if (status !== 200) throw new Error(`status ${status}`) - if (json.code !== 200) throw new Error(`code ${json.code}`) -}) - -await test("POST /api/task/upload/retry_failed works", async () => { - const { status, json } = await req("POST", "/api/task/upload/retry_failed") - if (status !== 200) throw new Error(`status ${status}`) - if (json.code !== 200) throw new Error(`code ${json.code}`) -}) - -await test("POST /api/task/upload/retry?tid= works", async () => { - const { status, json } = await req("POST", "/api/task/upload/retry?tid=1") - if (status !== 200) throw new Error(`status ${status}`) - if (json.code !== 200) throw new Error(`code ${json.code}`) -}) - -await test("POST /api/task/upload/cancel?tid= works", async () => { - const { status, json } = await req("POST", "/api/task/upload/cancel?tid=1") - if (status !== 200) throw new Error(`status ${status}`) - if (json.code !== 200) throw new Error(`code ${json.code}`) +await test("clear_done / clear_succeeded work", async () => { + for (const action of ["clear_done", "clear_succeeded"]) { + const { status, json } = await req("POST", `/api/task/copy/${action}`) + if (status !== 200) throw new Error(`${action}: status ${status}`) + if (json.code !== 200) throw new Error(`${action}: code ${json.code}`) + } }) -await test("POST /api/task/upload/delete?tid= works", async () => { - const { status, json } = await req("POST", "/api/task/upload/delete?tid=1") +await test("retry_failed works", async () => { + const { status, json } = await req("POST", "/api/task/copy/retry_failed") if (status !== 200) throw new Error(`status ${status}`) if (json.code !== 200) throw new Error(`code ${json.code}`) }) -await test("POST /api/task/upload/cancel_some works", async () => { - const { status, json } = await req("POST", "/api/task/upload/cancel_some", [ - "1", - "2", - ]) - if (status !== 200) throw new Error(`status ${status}`) - if (json.code !== 200) throw new Error(`code ${json.code}`) +await test("single-task operations return 404 for a missing tid", async () => { + for (const action of ["info", "cancel", "delete", "retry"]) { + const { status } = await req("POST", `/api/task/copy/${action}?tid=missing`) + if (status !== 404) throw new Error(`${action}: status ${status}`) + } }) -await test("POST /api/task/upload/delete_some works", async () => { - const { status, json } = await req("POST", "/api/task/upload/delete_some", [ - "1", - "2", - ]) - if (status !== 200) throw new Error(`status ${status}`) - if (json.code !== 200) throw new Error(`code ${json.code}`) +await test("bulk operations report per-tid errors", async () => { + for (const action of ["cancel_some", "delete_some", "retry_some"]) { + const { status, json } = await req("POST", `/api/task/copy/${action}`, [ + "missing-1", + "missing-2", + ]) + if (status !== 200) throw new Error(`${action}: status ${status}`) + if (json.data?.["missing-1"] !== "task not found") { + throw new Error(`${action}: unexpected payload ${JSON.stringify(json.data)}`) + } + } }) -await test("POST /api/task/upload/retry_some works", async () => { - const { status, json } = await req("POST", "/api/task/upload/retry_some", [ - "1", - "2", - ]) - if (status !== 200) throw new Error(`status ${status}`) - if (json.code !== 200) throw new Error(`code ${json.code}`) +await test("unauthenticated task API returns 401", async () => { + const res = await app.request("/api/task/copy/undone", {}, env) + if (res.status !== 401) throw new Error(`status ${res.status}`) }) console.log("\nAll task API tests completed.") diff --git a/src/backend/durable-objects/TaskRunner.ts b/src/backend/durable-objects/TaskRunner.ts new file mode 100644 index 00000000..f88b8930 --- /dev/null +++ b/src/backend/durable-objects/TaskRunner.ts @@ -0,0 +1,526 @@ +/** + * TaskRunner：任务队列的 Durable Object（Cloudflare 上的主执行器）。 + * + * 为什么用 DO： + * - 强一致 + 串行化：任务状态不需要跨实例 CAS/租约； + * - 内置 SQLite：进度按行更新，不必重写整份文档； + * - alarm：入队即可 setAlarm(now) 立即开跑，跑完再排下一次，几乎连续执行； + * alarm 处理器墙钟上限 15 分钟，足够覆盖多个 tick。 + * + * 结构： + * - SQLite `tasks` 表是唯一真相； + * - alarm() / tick() 调同一个 runner（与无 DO 平台共用的代码）； + * - 对 Worker 暴露 RPC（enqueue/list/get/patch/remove/tick）。 + */ +import { setEnvCtx } from "../internal/model/db" +import { getTaskSettings, resolveTickBudget } from "../internal/task/settings" +import { runTick } from "../internal/task/runner" +import type { + TaskClaimQuery, + TaskListQuery, + TaskRunnerStore, +} from "../internal/task/store" +import { pruneDoneTasks } from "../internal/task/store" +import type { TaskRecord, TaskType } from "../internal/task/types" +import { DONE_STATES, TASK_STATE } from "../internal/task/types" +import type { TaskRuntime } from "../internal/op/transfer" + +const COLUMNS = [ + "id", + "type", + "name", + "creator_id", + "creator", + "creator_role", + "state", + "status", + "progress", + "total_bytes", + "done_bytes", + "start_time", + "end_time", + "error", + "retry", + "max_retry", + "created_at", + "updated_at", + "next_run_at", + "attempts", + "verified", + "cursor", + "payload", + "lease_owner", + "lease_until", +] as const + +/** 允许通过 patch() 更新的列（白名单，避免把 id/type 改坏） */ +const PATCHABLE = new Set([ + "state", + "status", + "progress", + "total_bytes", + "done_bytes", + "start_time", + "end_time", + "error", + "retry", + "max_retry", + "updated_at", + "next_run_at", + "attempts", + "verified", + "cursor", + "payload", + "lease", +]) + +function parseJson(value: any, fallback: any): any { + if (value === null || value === undefined || value === "") return fallback + if (typeof value !== "string") return value + try { + return JSON.parse(value) + } catch { + return fallback + } +} + +function rowToRecord(row: any): TaskRecord { + return { + id: String(row.id), + type: String(row.type) as TaskType, + name: String(row.name || ""), + creator_id: Number(row.creator_id) || 0, + creator: String(row.creator || ""), + creator_role: Number(row.creator_role ?? -1), + state: Number(row.state) || 0, + status: String(row.status || ""), + progress: Number(row.progress) || 0, + total_bytes: Number(row.total_bytes) || 0, + done_bytes: Number(row.done_bytes) || 0, + start_time: row.start_time ?? null, + end_time: row.end_time ?? null, + error: String(row.error || ""), + retry: Number(row.retry) || 0, + max_retry: Number(row.max_retry) || 0, + created_at: Number(row.created_at) || 0, + updated_at: Number(row.updated_at) || 0, + next_run_at: Number(row.next_run_at) || 0, + attempts: Number(row.attempts) || 0, + verified: Boolean(row.verified), + cursor: parseJson(row.cursor, null), + payload: parseJson(row.payload, null), + lease: + row.lease_owner || row.lease_until + ? { + owner: String(row.lease_owner || ""), + expires_at: Number(row.lease_until) || 0, + } + : null, + } +} + +/** SQLite 存储实现（只在 DO 内使用） */ +class SqliteTaskStore implements TaskRunnerStore { + constructor( + private readonly sql: any, + private readonly env: any, + ) {} + + private exec(query: string, ...params: any[]): any { + return this.sql.exec(query, ...params) + } + + private rows(query: string, ...params: any[]): any[] { + return this.exec(query, ...params).toArray() + } + + async enqueue(record: TaskRecord): Promise { + this.exec( + `INSERT OR REPLACE INTO tasks (${COLUMNS.join(",")}) VALUES (${COLUMNS.map(() => "?").join(",")})`, + record.id, + record.type, + record.name, + record.creator_id, + record.creator, + record.creator_role, + record.state, + record.status, + record.progress, + record.total_bytes, + record.done_bytes, + record.start_time, + record.end_time, + record.error, + record.retry, + record.max_retry, + record.created_at, + record.updated_at, + record.next_run_at, + record.attempts, + record.verified ? 1 : 0, + record.cursor === undefined ? null : JSON.stringify(record.cursor), + record.payload === undefined ? null : JSON.stringify(record.payload), + record.lease?.owner ?? null, + record.lease?.expires_at ?? null, + ) + } + + async list(query: TaskListQuery): Promise { + const clauses = ["type = ?"] + const params: any[] = [query.type] + if (query.states?.length) { + clauses.push(`state IN (${query.states.map(() => "?").join(",")})`) + params.push(...query.states) + } + if (query.creator !== undefined && query.creator !== null) { + clauses.push("creator_id = ?") + params.push(query.creator) + } + const limit = Math.min(Math.max(1, query.limit ?? 200), 500) + params.push(limit) + return this.rows( + `SELECT * FROM tasks WHERE ${clauses.join(" AND ")} ORDER BY created_at DESC LIMIT ?`, + ...params, + ).map(rowToRecord) + } + + async get(_type: TaskType, id: string): Promise { + const row = this.rows("SELECT * FROM tasks WHERE id = ?", id)[0] + return row ? rowToRecord(row) : null + } + + async patch( + _type: TaskType, + id: string, + patch: Partial, + ): Promise { + const sets: string[] = [] + const params: any[] = [] + for (const [key, value] of Object.entries(patch)) { + if (!PATCHABLE.has(key)) continue + if (key === "lease") { + sets.push("lease_owner = ?", "lease_until = ?") + params.push((value as any)?.owner ?? null, (value as any)?.expires_at ?? null) + continue + } + sets.push(`${key} = ?`) + if (key === "cursor" || key === "payload") { + params.push(value === undefined ? null : JSON.stringify(value)) + } else if (key === "verified") { + params.push(value ? 1 : 0) + } else { + params.push(value ?? null) + } + } + if (sets.length === 0) return this.get(_type, id) + params.push(id) + const row = this.exec( + `UPDATE tasks SET ${sets.join(", ")} WHERE id = ? RETURNING *`, + ...params, + ).toArray()[0] + return row ? rowToRecord(row) : null + } + + async remove(_type: TaskType, id: string): Promise { + this.exec("DELETE FROM tasks WHERE id = ?", id) + } + + async removeMany(entries: Array<{ type: TaskType; id: string }>): Promise { + for (const entry of entries) { + this.exec("DELETE FROM tasks WHERE id = ?", entry.id) + } + } + + async countActive(creatorId?: number): Promise { + const params: any[] = [TASK_STATE.succeeded, TASK_STATE.canceled, TASK_STATE.failed] + let query = "SELECT COUNT(*) AS c FROM tasks WHERE state NOT IN (?,?,?)" + if (creatorId !== undefined && creatorId !== null) { + query += " AND creator_id = ?" + params.push(creatorId) + } + return Number(this.rows(query, ...params)[0]?.c) || 0 + } + + async prune(type: TaskType, keepDone: number): Promise { + const done = this.rows( + "SELECT * FROM tasks WHERE type = ? AND state IN (?,?,?) ORDER BY updated_at DESC", + type, + TASK_STATE.succeeded, + TASK_STATE.canceled, + TASK_STATE.failed, + ).map(rowToRecord) + if (done.length <= keepDone) return + const active = this.rows( + "SELECT * FROM tasks WHERE type = ? AND state NOT IN (?,?,?)", + type, + TASK_STATE.succeeded, + TASK_STATE.canceled, + TASK_STATE.failed, + ).map(rowToRecord) + const keep = new Set(pruneDoneTasks([...active, ...done], keepDone).map((r) => r.id)) + for (const record of done) { + if (!keep.has(record.id)) { + this.exec("DELETE FROM tasks WHERE id = ?", record.id) + } + } + } + + async claim(query: TaskClaimQuery): Promise { + for (let attempt = 0; attempt < 5; attempt++) { + const candidates = this.rows( + `SELECT * FROM tasks + WHERE (state = ? AND next_run_at <= ?) + OR (state = ? AND next_run_at <= ?) + OR (state = ? AND updated_at < ?) + ORDER BY CASE state WHEN ? THEN 0 WHEN ? THEN 1 ELSE 2 END, created_at ASC + LIMIT 1`, + TASK_STATE.pending, + query.now, + TASK_STATE.waitingRetry, + query.now, + TASK_STATE.running, + query.now - query.staleMs, + TASK_STATE.pending, + TASK_STATE.waitingRetry, + ) + const row = candidates[0] + if (!row) return null + const candidate = rowToRecord(row) + // running 且已超时 = 上次执行被杀。累计 attempts，超过阈值直接判失败， + // 避免一个永远跑不完的任务把整个队列卡住。 + if (candidate.state === TASK_STATE.running) { + const attempts = (candidate.attempts || 0) + 1 + if (attempts > 3) { + this.exec( + "UPDATE tasks SET state = ?, status = ?, error = ?, end_time = ?, updated_at = ?, lease_owner = NULL, lease_until = NULL WHERE id = ?", + TASK_STATE.failed, + "stalled", + `stalled: no completion after ${attempts} attempts (last update ${new Date(candidate.updated_at).toISOString()})`, + new Date(query.now).toISOString(), + query.now, + candidate.id, + ) + continue + } + return this.markRunning(candidate, query, attempts) + } + return this.markRunning(candidate, query, candidate.attempts || 0) + } + return null + } + + private markRunning( + candidate: TaskRecord, + query: TaskClaimQuery, + attempts: number, + ): TaskRecord | null { + const row = this.exec( + `UPDATE tasks SET state = ?, start_time = COALESCE(start_time, ?), updated_at = ?, + attempts = ?, lease_owner = ?, lease_until = ? + WHERE id = ? RETURNING *`, + TASK_STATE.running, + candidate.start_time || new Date(query.now).toISOString(), + query.now, + attempts, + query.owner, + query.now + query.leaseMs, + candidate.id, + ).toArray()[0] + return row ? rowToRecord(row) : null + } +} + +export class TaskRunner { + private state: any + private env: any + private storeRef: SqliteTaskStore | null = null + private ready = false + private noProgressStreak = 0 + + constructor(state: any, env: any) { + this.state = state + this.env = env + } + + private get sql(): any { + return (this.state.storage as any).sql + } + + private ensureSchema(): void { + if (this.ready) return + this.sql.exec(` + CREATE TABLE IF NOT EXISTS tasks ( + id TEXT PRIMARY KEY, + type TEXT NOT NULL, + name TEXT NOT NULL DEFAULT '', + creator_id INTEGER NOT NULL DEFAULT 0, + creator TEXT NOT NULL DEFAULT '', + creator_role INTEGER NOT NULL DEFAULT -1, + state INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT '', + progress REAL NOT NULL DEFAULT 0, + total_bytes INTEGER NOT NULL DEFAULT 0, + done_bytes INTEGER NOT NULL DEFAULT 0, + start_time TEXT, + end_time TEXT, + error TEXT NOT NULL DEFAULT '', + retry INTEGER NOT NULL DEFAULT 0, + max_retry INTEGER NOT NULL DEFAULT 2, + created_at INTEGER NOT NULL DEFAULT 0, + updated_at INTEGER NOT NULL DEFAULT 0, + next_run_at INTEGER NOT NULL DEFAULT 0, + attempts INTEGER NOT NULL DEFAULT 0, + verified INTEGER NOT NULL DEFAULT 0, + cursor TEXT, + payload TEXT, + lease_owner TEXT, + lease_until INTEGER + ) + `) + this.sql.exec( + "CREATE INDEX IF NOT EXISTS idx_tasks_state ON tasks (state, next_run_at)", + ) + this.sql.exec( + "CREATE INDEX IF NOT EXISTS idx_tasks_type_state ON tasks (type, state, created_at)", + ) + this.sql.exec( + "CREATE INDEX IF NOT EXISTS idx_tasks_creator ON tasks (creator_id, state)", + ) + this.ready = true + } + + private store(): SqliteTaskStore { + this.ensureSchema() + if (!this.storeRef) { + this.storeRef = new SqliteTaskStore(this.sql, this.env) + } + return this.storeRef + } + + private runtime(): TaskRuntime { + return { + env: this.env, + origin: this.env?.PUBLIC_ORIGIN, + } + } + + // ───────────────────────── RPC（Worker 调用） ───────────────────────── + + async enqueue(record: TaskRecord): Promise { + setEnvCtx(this.env) + this.store().enqueue(record) + // 只有「待执行」的记录需要唤醒 alarm：同步完成的历史记录不排期， + // 否则每次小文件复制都会白醒一次 DO（免费计划的行写/请求配额很紧）。 + const actionable = + record.state === TASK_STATE.pending || + record.state === TASK_STATE.waitingRetry || + record.state === TASK_STATE.running + if (!actionable) return + const current = await this.state.storage.getAlarm() + if (current === null || current > Date.now() + 1000) { + await this.state.storage.setAlarm(Date.now()) + } + } + + async list(query: TaskListQuery): Promise { + setEnvCtx(this.env) + return this.store().list(query) + } + + async get(type: TaskType, id: string): Promise { + setEnvCtx(this.env) + return this.store().get(type, id) + } + + async patch( + type: TaskType, + id: string, + patch: Partial, + ): Promise { + setEnvCtx(this.env) + return this.store().patch(type, id, patch) + } + + async remove(type: TaskType, id: string): Promise { + setEnvCtx(this.env) + await this.store().remove(type, id) + } + + async removeMany(entries: Array<{ type: TaskType; id: string }>): Promise { + setEnvCtx(this.env) + await this.store().removeMany(entries) + } + + async countActive(creatorId?: number): Promise { + setEnvCtx(this.env) + return this.store().countActive(creatorId) + } + + async prune(type: TaskType, keepDone: number): Promise { + setEnvCtx(this.env) + await this.store().prune(type, keepDone) + } + + /** + * 跑一个 tick。`budgetMs` 可由调用方显式指定（/api/task/run 支持）， + * 否则按设置/泵的默认值。 + */ + async tick( + options: { budgetMs?: number } = {}, + ): Promise<{ ran: number; remaining: number; progressed: boolean }> { + setEnvCtx(this.env) + const settings = await getTaskSettings() + if (!settings.enabled) { + return { ran: 0, remaining: 0, progressed: false } + } + const budgetMs = + options.budgetMs && options.budgetMs > 0 + ? Math.min(60000, Math.max(200, options.budgetMs)) + : resolveTickBudget(settings, "do") + const result = await runTick(this.store(), this.runtime(), { + budgetMs, + concurrency: settings.parallelUnits, + owner: `do:${Date.now()}:${Math.random().toString(36).slice(2, 8)}`, + }) + + if (result.remaining > 0) { + if (result.progressed) this.noProgressStreak = 0 + else this.noProgressStreak += 1 + const delay = result.progressed + ? 50 + : Math.min(60000, 1000 * Math.pow(2, Math.min(6, this.noProgressStreak))) + await this.state.storage.setAlarm(Date.now() + delay) + } else { + this.noProgressStreak = 0 + try { + await this.state.storage.deleteAlarm() + } catch { + // deleteAlarm 在 alarm 处理器内可能失败，属正常 + } + } + return result + } + + // ───────────────────────── Alarm ───────────────────────── + + async alarm(): Promise { + try { + await this.tick({}) + } catch (err) { + // alarm 的自动重试只有 6 次；这里自己吞掉并重新排期，保证长跑任务不会 + // 因为一次下游抖动就永久停摆。 + console.error("[TaskRunner] alarm tick failed:", err) + try { + await this.state.storage.setAlarm(Date.now() + 30000) + } catch {} + } + } + + /** 非 RPC 入口的兜底（某些平台会以 HTTP 方式探测） */ + async fetch(): Promise { + return new Response( + JSON.stringify({ code: 200, message: "task runner", data: null }), + { headers: { "content-type": "application/json" } }, + ) + } +} diff --git a/src/backend/internal/model/db.ts b/src/backend/internal/model/db.ts index e07a9c36..19778e20 100644 --- a/src/backend/internal/model/db.ts +++ b/src/backend/internal/model/db.ts @@ -1007,6 +1007,102 @@ export const defaultDb = { group: 12, flag: 1, }, + // ── 任务系统（group 11 = TRAFFIC，与 Go 版任务设置同组） ── + // TS 端用「时间预算」而不是线程数：Worker 的执行以 tick 为单位， + // 每次 tick 能跑多久直接决定吞吐与是否触发 CPU 超限。 + { + key: "task_enabled", + value: "true", + type: "bool", + help: + "Enable the persistent task queue. When disabled, large file operations run inline (old behaviour) and no task history is recorded.", + group: 11, + flag: 1, + }, + { + key: "task_sync_threshold_mb", + value: "32", + type: "number", + help: + "Operations whose total size is at or below this value run synchronously; larger ones become background tasks. 0 means always async.", + group: 11, + flag: 1, + }, + { + key: "task_probe_max_entries", + value: "200", + type: "number", + help: + "Maximum number of entries probed when deciding sync vs async. Exceeding it always switches to a background task.", + group: 11, + flag: 1, + }, + { + key: "task_tick_budget_ms", + value: "0", + type: "number", + help: + "Time budget per execution tick in milliseconds (200-60000). 0 = automatic (2000ms on Cloudflare, conservative for the free plan). Paid plans should raise this to 20000.", + group: 11, + flag: 1, + }, + { + key: "task_parallel_units", + value: "1", + type: "number", + help: + "Number of tasks executed concurrently per tick (1-4). Cloudflare allows at most 6 outgoing connections per invocation; each transfer usually uses 2.", + group: 11, + flag: 1, + }, + { + key: "task_keep_done", + value: "50", + type: "number", + help: "How many finished tasks to keep per type. Older ones are pruned.", + group: 11, + flag: 1, + }, + { + key: "task_copy_max_retry", + value: "2", + type: "number", + help: "Maximum retries for copy tasks", + group: 11, + flag: 1, + }, + { + key: "task_move_max_retry", + value: "2", + type: "number", + help: "Maximum retries for move tasks", + group: 11, + flag: 1, + }, + { + key: "task_decompress_max_retry", + value: "2", + type: "number", + help: "Maximum retries for decompress tasks", + group: 11, + flag: 1, + }, + { + key: "task_download_max_retry", + value: "2", + type: "number", + help: "Maximum retries for offline download tasks", + group: 11, + flag: 1, + }, + { + key: "task_seed_max_retry", + value: "2", + type: "number", + help: "Maximum retries for seed hashing/transfer tasks", + group: 11, + flag: 1, + }, ], storages: [], users: [ diff --git a/src/backend/internal/model/store/backend.ts b/src/backend/internal/model/store/backend.ts index c2e91b57..b7e75591 100644 --- a/src/backend/internal/model/store/backend.ts +++ b/src/backend/internal/model/store/backend.ts @@ -731,13 +731,45 @@ export async function getStoreBackend(env?: any): Promise { return { name: driver.name, load: (e?: any) => format.load(driver, e), - save: (data: any, e?: any) => format.save(data, driver, e), + save: (data: any, e?: any) => + format.save(data, driver, e), isConfigured: (e?: any) => driver.isAvailable(e), init: (e?: any) => driver.init(e), health: (e?: any) => driver.health(e), } } +/** + * 原始键值访问器（供任务文档等「配置之外」的数据使用）。 + * + * 为什么不复用 `db.ts` 的 getDb/saveDb：那份数据是「整份配置对象」，任何一次 + * saveDb 都会重写 storages/users/settings 全量内容。任务进度是高频小写入， + * 若并入配置对象，两个 isolate 并发写时必然互相覆盖（一个写设置、一个写进度， + * 后写的会把先写的整份覆盖）。这里复用同一套 Driver，但使用独立的 key 空间。 + * + * 注意：`list` 在 blob 驱动上意义有限，任务文档使用固定 key，不依赖它。 + */ +export interface KvAccessor { + /** 底层驱动名（blob/kv/cfkv/d1/do/mysql） */ + name: string + get(key: string, env?: any): Promise + put(key: string, value: string, env?: any): Promise + delete(key: string, env?: any): Promise + list(prefix: string, env?: any): Promise +} + +export async function getKvAccessor(env?: any): Promise { + const { driver } = await getStorageBackend(env) + return { + name: driver.name, + get: (key: string, e?: any) => driver.get(key, e ?? env), + put: (key: string, value: string, e?: any) => + driver.put(key, value, e ?? env), + delete: (key: string, e?: any) => driver.delete(key, e ?? env), + list: (prefix: string, e?: any) => driver.list(prefix, e ?? env), + } +} + /** * 当前后端的健康/连接状态，用于 /debug/info 与 /admin/kv/status。 * diff --git a/src/backend/internal/model/store/driver/kv.test.ts b/src/backend/internal/model/store/driver/kv.test.ts index e1b7dda3..a46081d5 100644 --- a/src/backend/internal/model/store/driver/kv.test.ts +++ b/src/backend/internal/model/store/driver/kv.test.ts @@ -1,6 +1,7 @@ import assert from "node:assert/strict" import test from "node:test" import { kvDriver } from "./kv" +import { KEY_INDEX_KEY } from "../keycodec" /** * issue #125 回归：binding 模式 list() 必须同时兼容两种平台语义。 @@ -144,6 +145,28 @@ test("kv health (binding, key format) exercises list so exhausted quota is not r assert.equal(listCalls, 1) }) +test("kv health (binding, key format) skips list when the key index exists", async () => { + // 索引存在时 key 格式的稳态读写只依赖 get，健康检查不应再白白消耗 + // 1 次 list 配额（issue #125：list 配额只有 1000 次/天）。 + let listCalls = 0 + const health = await kvDriver.health({ + DB_FORMAT: "key", + KV: { + get: async (key: string) => + key === KEY_INDEX_KEY ? '{"v":1,"keys":["users_1"]}' : null, + put: async () => {}, + delete: async () => {}, + list: async () => { + listCalls += 1 + return { keys: [], list_complete: true } + }, + }, + }) + + assert.equal(health.available, true) + assert.equal(listCalls, 0) +}) + test("kv health (binding, map format) does not require an unsupported list API", async () => { // ESA EdgeKV exposes get/put/delete but no list. Map format only needs the // single openlist_config key, so it must still be considered healthy. diff --git a/src/backend/internal/model/store/driver/kv.ts b/src/backend/internal/model/store/driver/kv.ts index 6dfc3423..7a5d0d6d 100644 --- a/src/backend/internal/model/store/driver/kv.ts +++ b/src/backend/internal/model/store/driver/kv.ts @@ -9,6 +9,7 @@ */ import { sanitizeProxyOrigin } from "../proxy" import { KV_HEALTH_PROBE_KEY, readFormat } from "../config" +import { KEY_INDEX_KEY } from "../keycodec" import type { Driver, EnvContext } from "../types" /** @@ -295,6 +296,50 @@ function requireProxyBaseUrl(env?: EnvContext): string { return baseUrl } +/** + * 读取 binding 的文本值并归一化为 `string | null`。 + * + * 只从 binding 读取，不涉及代理分支。抽成独立函数的原因： + * `get()` 与 `health()` 都需要「兼容两种 get 签名」，各写一份迟早漂移。 + * + * Cloudflare KV 用 `get(key, "text")`，EdgeOne KV 用 `get(key, {type:"text"})`， + * 两者签名不兼容，需按序尝试并归一化返回值（避免拿到对象导致上游 JSON.parse 失败）。 + * + * 但要区分「签名不兼容」与「真实故障」： + * - 签名不兼容会抛 TypeError（参数类型不符），此时才应回退到第二种签名； + * - 网络/权限错误（401、超时等）必须**原样抛出**，否则会被第二次尝试 + * 掩盖成 "key not found"，让调用方误判为数据不存在而写入错误状态。 + */ +async function kvGetText(kv: any, key: string): Promise { + const isSignatureError = (e: any): boolean => + e instanceof TypeError || + /not a function|invalid|unexpected|argument/i.test(String(e?.message || "")) + + let value: any + try { + value = await kv.get(key, "text") + } catch (e) { + if (!isSignatureError(e)) throw e + value = undefined + } + + if (value === undefined || value === null) { + try { + value = await kv.get(key, { type: "text" }) + } catch (e) { + // 若第一种签名已成功执行（未抛签名错误），说明第二种只是"空结果"的回退， + // 此时真实错误同样要抛出，不得吞掉。 + if (!isSignatureError(e)) throw e + value = null + } + } + + if (value === undefined || value === null) return null + if (typeof value === "string") return value + // 绑定误返回对象时统一序列化，保持 Driver.get 的 string 契约 + return JSON.stringify(value) +} + export const kvDriver: Driver = { name: "kv", @@ -326,42 +371,7 @@ export const kvDriver: Driver = { // 模式1: Binding 模式 if (kv) { - // Cloudflare KV 用 get(key, "text")，EdgeOne KV 用 get(key, {type:"text"})， - // 两者签名不兼容，需按序尝试并归一化返回值（避免拿到对象导致上游 JSON.parse 失败）。 - // - // 但要区分「签名不兼容」与「真实故障」： - // - 签名不兼容会抛 TypeError（参数类型不符），此时才应回退到第二种签名； - // - 网络/权限错误（401、超时等）必须**原样抛出**，否则会被第二次尝试 - // 掩盖成 "key not found"，让调用方误判为数据不存在而写入错误状态。 - const isSignatureError = (e: any): boolean => - e instanceof TypeError || - /not a function|invalid|unexpected|argument/i.test( - String(e?.message || ""), - ) - - let value: any - try { - value = await kv.get(key, "text") - } catch (e) { - if (!isSignatureError(e)) throw e - value = undefined - } - - if (value === undefined || value === null) { - try { - value = await kv.get(key, { type: "text" }) - } catch (e) { - // 若第一种签名已成功执行（未抛签名错误），说明第二种只是"空结果"的回退， - // 此时真实错误同样要抛出，不得吞掉。 - if (!isSignatureError(e)) throw e - value = null - } - } - - if (value === undefined || value === null) return null - if (typeof value === "string") return value - // 绑定误返回对象时统一序列化，保持 Driver.get 的 string 契约 - return JSON.stringify(value) + return kvGetText(kv, key) } // 模式2: HTTP 代理模式（无原生 binding → 经 Edge Function 代理） @@ -564,11 +574,12 @@ export const kvDriver: Driver = { } else { await kv.get(KV_HEALTH_PROBE_KEY) } - // key 格式的 load/save 依赖 list；map 格式只读写固定的 openlist_config, - // 不要求 EdgeKV 暴露 list API（ESA EdgeKV 内置 API 仅提供 get/put/delete）。 - // 只有 key 格式才探测 list，避免 map 格式部署被不支持的 list() 误判为 - // 存储不可用（issue #159）。对 key 格式仍检查 list，防止配额耗尽/接口异常 - // 导致 env_check 误报 ready（issue #125）。使用专用前缀，取回空页只耗 1 次。 + // key 格式的稳态读写只依赖 get/put + 全局索引（issue #125）： + // - 索引键存在时：一次 load 只花 1 次索引 get + N 次记录 get，不需要 list； + // - 索引缺失时（旧部署首次升级 / 超大索引）：load/save 会回退 list， + // 此时必须确认 list 可用，否则初始化会 500。 + // 因此这里先读索引键（1 次 read，比 list 便宜两个数量级），只有索引缺失 + // 才真的调用 list 探测。map 格式不读写索引，完全不涉及 list。 // 直接复用后端的 readFormat()：健康检查与真实读写必须用同一份解析逻辑 // （env 优先级、缺省 map、DB_DRIVER=json 别名），复刻一份迟早漂移。 if (readFormat(env) === "key") { @@ -577,7 +588,14 @@ export const kvDriver: Driver = { "KV binding does not support list(), which is required by DB_FORMAT=key", ) } - await kv.list({ prefix: KV_HEALTH_PROBE_KEY, cursor: "", limit: 256 }) + const indexRaw = await kvGetText(kv, KEY_INDEX_KEY) + if (indexRaw === null) { + await kv.list({ + prefix: KV_HEALTH_PROBE_KEY, + cursor: "", + limit: 256, + }) + } } return { driver: "kv", diff --git a/src/backend/internal/model/store/format/key.ts b/src/backend/internal/model/store/format/key.ts index 6857706b..986ddb7a 100644 --- a/src/backend/internal/model/store/format/key.ts +++ b/src/backend/internal/model/store/format/key.ts @@ -16,13 +16,55 @@ * * 转义方案无需反向解析：所有读写都是"构造键名"，实体的主键值一律 * 来自记录 JSON 本身，key 只用作存储地址。 + * + * ## 全局索引（issue #125） + * + * 历史实现每次 load/save 都靠 `list()` 枚举键名，即使 6 张表合并成 1 次 + * 全局 list，Cloudflare 免费版 1000 次/天的 list 配额也只够约 1000 次 + * load/save —— 页面刷新几次就见底（且大库一次 list 还可能要翻多页）。 + * + * 现在把「当前存活的实体键」写入 `KEY_INDEX_KEY`（见 keycodec）： + * - load：1 次 `get` 索引 + N 次 `get` 记录，不再消耗 list； + * - save：写记录 → 写新索引 → 按旧索引删除陈旧键，同样不消耗 list； + * - 索引缺失（旧部署首次升级 / 索引被删）时回退到 1 次 list 并**补写索引**， + * 因此升级无需手工迁移，之后也不再重复 list； + * - 索引损坏或超过单值上限（EdgeOne KV 1 MB）时不使用它并回退 list， + * 宁可多耗一次 list，也不能让读取看不到数据。 + * + * 索引是「成员名单」而非数据本身：记录值仍以实体键为唯一权威。写入顺序 + * 刻意是「先写记录 → 再写索引 → 最后删陈旧键」，任一步失败都不会让 + * 已经成功写入的记录从索引中消失（最坏情况只是残留几个无引用的旧键）。 */ import type { FormatAdapter, Driver } from "../types" import { TABLE_NAMES, TABLE_KEY, type TableName } from "../schema" -import { entityKeyOf, tableKeyPrefix } from "../keycodec" +import { entityKeyOf, tableKeyPrefix, KEY_INDEX_KEY } from "../keycodec" const INIT_MARK = "openlist_config" +/** 索引文档版本：结构变化时递增，旧版本会按「无索引」回退到 list。 */ +const INDEX_VERSION = 1 + +/** + * 索引值的字节上限。 + * + * EdgeOne KV 单个 value 最大 1 MB，Cloudflare 为 25 MiB；取 900 KB 留出 + * 余量。索引里的键名均为 ASCII（keycodec 保证），因此 length 近似等于字节数。 + * 超过上限时放弃索引：大库继续走 list，行为与升级前一致。 + */ +const INDEX_MAX_BYTES = 900 * 1024 + +/** + * 记录读写的并发度。 + * + * key 格式一次 load 要逐条 get，默认库就有 100+ 条记录；串行执行在边缘 + * 网络上会把单次请求拖到数秒。并发 12 把往返压缩到约 1/10，同时远低于 + * Workers 的子请求上限。 + */ +const IO_CONCURRENCY = 12 + +/** 超大索引告警只打印一次，避免每次 load/save 刷屏。 */ +let oversizeWarned = false + /** 构造表前缀（使用共享键名编码，兼容 EdgeOne KV 字符集约束） */ function tablePrefix(table: string): string { return tableKeyPrefix(table) @@ -33,23 +75,142 @@ function fullKeyOf(table: string, id: string): string { return entityKeyOf(table, id) } +/** + * 以受限并发执行异步任务，结果顺序与输入一致。 + * + * 不使用 `Promise.all(items.map(fn))`：记录数可能上千，一次性并发全部 + * 子请求会撞上平台限制；也不串行，否则每次 load 的耗时随记录数线性增长。 + */ +async function runBounded( + items: readonly T[], + limit: number, + fn: (item: T, index: number) => Promise, +): Promise { + const results = new Array(items.length) + let cursor = 0 + const workers = Array.from( + { length: Math.min(Math.max(1, limit), items.length) }, + async () => { + while (true) { + const index = cursor++ + if (index >= items.length) return + results[index] = await fn(items[index], index) + } + }, + ) + await Promise.all(workers) + return results +} + +/** + * 读取全局索引。 + * + * 任何「读不到 / 解析失败 / 结构不识别 / 版本不符」都返回 null，由调用方 + * 回退到 list。索引只是加速缓存，绝不能因为它损坏就返回空数据库 —— + * 那会被上层当成「全新部署」，进而用空库覆盖真实数据。 + */ +async function readIndex(driver: Driver, env?: any): Promise { + let raw: string | null + try { + raw = await driver.get(KEY_INDEX_KEY, env) + } catch (err) { + console.warn( + "[keyFormat] failed to read key index; falling back to list:", + err, + ) + return null + } + if (!raw) return null + + try { + const parsed: any = JSON.parse(raw) + if ( + parsed && + typeof parsed === "object" && + parsed.v === INDEX_VERSION && + Array.isArray(parsed.keys) && + parsed.keys.every((k: any) => typeof k === "string" && k.length > 0) + ) { + return parsed.keys as string[] + } + console.warn( + "[keyFormat] key index has an unknown shape; falling back to list", + ) + } catch (err) { + console.warn( + "[keyFormat] key index is not valid JSON; falling back to list:", + err, + ) + } + return null +} + +/** + * 写入全局索引。 + * + * @returns true 表示索引已落盘；false 表示索引过大被跳过（此时会删除旧索引， + * 否则下一次 load 会读到「缺少新键」的旧索引而看不到新数据）。 + */ +async function writeIndex( + driver: Driver, + keys: string[], + env?: any, +): Promise { + const payload = JSON.stringify({ v: INDEX_VERSION, keys }) + + if (payload.length > INDEX_MAX_BYTES) { + if (!oversizeWarned) { + oversizeWarned = true + console.warn( + `[keyFormat] key index is ${payload.length} bytes, over the ` + + `${INDEX_MAX_BYTES}-byte limit; falling back to list() for this ` + + `deployment. Reduce record count or use DB_FORMAT=map/sql.`, + ) + } + // 关键：旧索引必须删除，否则它会成为一份「永远不会被修正」的过期名单。 + try { + await driver.delete(KEY_INDEX_KEY, env) + } catch (err) { + console.warn("[keyFormat] failed to drop stale key index:", err) + } + return false + } + + await driver.put(KEY_INDEX_KEY, payload, env) + return true +} + +/** + * 读取当前存活键：优先索引，缺失时回退 list。 + * + * `fromIndex` 用于决定是否需要补写索引；即使索引存在，返回的键也可能 + * 含已删除实体（删除阶段失败），后续 get 会得到 null 并被跳过。 + */ +async function readMembership( + driver: Driver, + env?: any, +): Promise<{ keys: string[]; fromIndex: boolean }> { + const indexed = await readIndex(driver, env) + if (indexed) return { keys: indexed, fromIndex: true } + return { keys: await driver.list("", env), fromIndex: false } +} + /** * 将一次全局 list 的结果按表前缀分组。 * - * key 格式的键名统一为 `_`（见 keycodec），因此一次空前缀 - * list 即可取回全部表的键，再按前缀分拣 —— 结果与逐表 list 完全一致， - * 但把 load/save 各自的 list 消耗从「每表 1 次（6 张表 = 6 次）」降为 - * 1 次。Cloudflare 免费版 list 配额仅 1000 次/天（issue #125）， - * key + kv 组合下这是可持续性的关键；对配额无限的平台也无害（省 5 次 - * 往返）。 + * 键名统一为 `
_`（见 keycodec），因此一次空前缀 list 即可取回 + * 全部表的键，再按前缀分拣 —— 结果与逐表 list 完全一致。排序保证记录顺序 + * 与 KV list 的字典序一致，不因索引建立/迁移而变化。 * - * 命名空间中不属于任何表前缀的键（如 INIT_MARK、其他应用的键）会被 - * 忽略，与旧实现「逐表前缀 list 天然排除它们」的行为一致。 + * 命名空间中不属于任何表前缀的键（如 INIT_MARK、索引键、其他应用的键） + * 会被忽略，与旧实现「逐表前缀 list 天然排除它们」的行为一致。 */ function groupKeysByTable(allKeys: string[]): Record { const out = {} as Record for (const table of TABLE_NAMES) out[table] = [] - for (const key of allKeys) { + + // 去重 + 排序：索引损坏/人工编辑可能产生重复键，重复 get 会返回重复记录。 + for (const key of [...new Set(allKeys)].sort()) { for (const table of TABLE_NAMES) { if (key.startsWith(tablePrefix(table))) { out[table].push(key) @@ -60,6 +221,29 @@ function groupKeysByTable(allKeys: string[]): Record { return out } +/** 并发读取并解析一批实体记录（空值跳过，损坏记录告警后跳过）。 */ +async function fetchRecords( + keys: string[], + driver: Driver, + env?: any, +): Promise { + const raws = await runBounded(keys, IO_CONCURRENCY, (key) => + driver.get(key, env), + ) + + const records: any[] = [] + for (let i = 0; i < raws.length; i++) { + const raw = raws[i] + if (!raw) continue + try { + records.push(JSON.parse(raw)) + } catch (err) { + console.warn(`[keyFormat] Failed to parse ${keys[i]}:`, err) + } + } + return records +} + export const keyFormat: FormatAdapter = { name: "key", @@ -68,25 +252,28 @@ export const keyFormat: FormatAdapter = { const mark = await driver.get(INIT_MARK, env) if (!mark) return null - // 一次全局 list 代替逐表 list（6 张表 = 6 次），见 groupKeysByTable - const grouped = groupKeysByTable(await driver.list("", env)) + const { keys, fromIndex } = await readMembership(driver, env) + const grouped = groupKeysByTable(keys) + + // 旧部署首次升级 / 索引被删：本次已付出 1 次 list，顺手把索引补上， + // 后续请求即可只走 get。补写失败不影响本次读取（下次再扫一次）。 + // 只收录实体键：索引键、INIT_MARK 与其他应用的键不进入名单。 + if (!fromIndex) { + const entityKeys = TABLE_NAMES.flatMap((table) => grouped[table]).sort() + try { + await writeIndex(driver, entityKeys, env) + } catch (err) { + console.warn( + "[keyFormat] failed to persist key index; next load will rescan:", + err, + ) + } + } const out: Record = {} for (const table of TABLE_NAMES) { - const records = [] - for (const key of grouped[table]) { - const raw = await driver.get(key, env) - if (raw) { - try { - records.push(JSON.parse(raw)) - } catch (err) { - console.warn(`[keyFormat] Failed to parse ${key}:`, err) - } - } - } - - out[table] = records + out[table] = await fetchRecords(grouped[table], driver, env) } return out @@ -102,7 +289,9 @@ export const keyFormat: FormatAdapter = { // // 改为先写后删后，失败时的最坏情况是**残留**旧键（可被下一次成功保存清理）， // 而不会丢数据；正常成功路径的最终状态与原来完全一致。 - const liveKeys = new Set() + // Map 天然去重：同一 id 出现多次时只保留最后一次，key 名也不会重复写。 + const writes = new Map() + for (const table of TABLE_NAMES) { const keyCol = TABLE_KEY[table] const records = data?.[table] || [] @@ -111,78 +300,111 @@ export const keyFormat: FormatAdapter = { const id = String(record?.[keyCol] ?? "") if (!id) continue - const key = fullKeyOf(table, id) - liveKeys.add(key) - const value = JSON.stringify(record) - await driver.put(key, value, env) + writes.set(fullKeyOf(table, id), JSON.stringify(record)) } } - // 清理本快照中已不存在的键（删除的实体 / 改名的键） - // 一次全局 list 代替逐表 list（6 张表 = 6 次），见 groupKeysByTable - const grouped = groupKeysByTable(await driver.list("", env)) + const liveKeys = new Set(writes.keys()) + + // 并发写记录：key 格式的默认库就有 100+ 条，串行写会明显拖慢保存。 + await runBounded([...writes], IO_CONCURRENCY, ([key, value]) => + driver.put(key, value, env), + ) + + // 旧成员名单：优先索引；索引缺失（首次保存/旧部署）才扫一次 list。 + const { keys: previousKeys } = await readMembership(driver, env) + + // 先写索引，再删陈旧键。顺序不能反： + // - 索引先写：并发读最多看到「新名单 + 旧残留键」，不会看到「旧名单 + 已删除键」； + // - 若索引写失败，删除旧索引并抛错，下一次 load 会 list 兜底看到全部新记录。 + try { + await writeIndex(driver, [...liveKeys].sort(), env) + } catch (err) { + try { + await driver.delete(KEY_INDEX_KEY, env) + } catch { + // 删除失败也只能作罢：下一次 load 的索引与记录不一致时仍能读到已有记录， + // 但新记录可能暂时不可见。此处不应掩盖原始写入错误。 + } + throw err + } + + // 清理本快照中已不存在的键（删除的实体 / 改名的键）。 + // 只处理属于已知表前缀的键，避免误删 INIT_MARK、索引键或其他应用的键。 + const stale: string[] = [] + const grouped = groupKeysByTable(previousKeys) for (const table of TABLE_NAMES) { for (const key of grouped[table]) { - if (liveKeys.has(key)) continue - await driver.delete(key, env) + if (!liveKeys.has(key)) stale.push(key) } } + await runBounded(stale, IO_CONCURRENCY, (key) => driver.delete(key, env)) // 标记已初始化 await driver.put(INIT_MARK, String(Date.now()), env) return true }, - + async getTable(table: string, driver: Driver, env?: any): Promise { const prefix = tablePrefix(table) - const keys = await driver.list(prefix, env) - - const records = [] - for (const key of keys) { - const raw = await driver.get(key, env) - if (raw) { - try { - records.push(JSON.parse(raw)) - } catch (err) { - console.warn(`[keyFormat] Failed to parse ${key}:`, err) - } - } - } + const indexed = await readIndex(driver, env) + // 有索引时直接过滤，不再 list（issue #125 的 list 配额问题） + const keys = indexed + ? indexed.filter((key) => key.startsWith(prefix)).sort() + : await driver.list(prefix, env) - return records + return fetchRecords(keys, driver, env) }, async saveTable( table: string, records: any[], driver: Driver, - env?: any + env?: any, ): Promise { const keyCol = TABLE_KEY[table as keyof typeof TABLE_KEY] const prefix = tablePrefix(table) - // 清空旧数据 - const oldKeys = await driver.list(prefix, env) - for (const key of oldKeys) { - await driver.delete(key, env) - } + const indexed = await readIndex(driver, env) + const oldKeys = indexed + ? indexed.filter((key) => key.startsWith(prefix)) + : await driver.list(prefix, env) - // 写入新数据 + // 同 save：先写新记录，再删改名前/已删除的旧键，避免失败时整表变空。 + const newKeys = new Set() + const writes: Array<{ key: string; value: string }> = [] for (const record of records) { const id = String(record?.[keyCol] ?? "") if (!id) continue const key = fullKeyOf(table, id) - const value = JSON.stringify(record) - await driver.put(key, value, env) + newKeys.add(key) + writes.push({ key, value: JSON.stringify(record) }) + } + + await runBounded(writes, IO_CONCURRENCY, (w) => + driver.put(w.key, w.value, env), + ) + + // 索引存在时同步替换该表的键；索引缺失就保持缺失，下一次 load 会 list 重建。 + if (indexed) { + const merged = indexed.filter((key) => !key.startsWith(prefix)) + merged.push(...newKeys) + await writeIndex(driver, merged.sort(), env) } + + await runBounded( + oldKeys.filter((key) => !newKeys.has(key)), + IO_CONCURRENCY, + (key) => driver.delete(key, env), + ) }, async getRecord( table: string, key: string, driver: Driver, - env?: any + env?: any, ): Promise { const fullKey = fullKeyOf(table, key) const raw = await driver.get(fullKey, env) @@ -201,20 +423,35 @@ export const keyFormat: FormatAdapter = { key: string, record: any, driver: Driver, - env?: any + env?: any, ): Promise { const fullKey = fullKeyOf(table, key) const value = JSON.stringify(record) await driver.put(fullKey, value, env) + + // 索引存在时必须同步加入，否则后续 load 会漏掉这条记录。 + const indexed = await readIndex(driver, env) + if (indexed && !indexed.includes(fullKey)) { + await writeIndex(driver, [...indexed, fullKey].sort(), env) + } }, async deleteRecord( table: string, key: string, driver: Driver, - env?: any + env?: any, ): Promise { const fullKey = fullKeyOf(table, key) await driver.delete(fullKey, env) + + const indexed = await readIndex(driver, env) + if (indexed?.includes(fullKey)) { + await writeIndex( + driver, + indexed.filter((k) => k !== fullKey), + env, + ) + } }, } diff --git a/src/backend/internal/model/store/keycodec.ts b/src/backend/internal/model/store/keycodec.ts index 93deea24..e27b0cdd 100644 --- a/src/backend/internal/model/store/keycodec.ts +++ b/src/backend/internal/model/store/keycodec.ts @@ -90,6 +90,21 @@ export function decodeKeyPart(input: string): string { return out } +/** + * key 格式的全局索引键。 + * + * key 格式把每张表摊平成 `
_` 多条记录，历史实现靠 `list()` 枚举键名； + * Cloudflare 免费版 list 配额只有 1000 次/天，而一次 load/save 就要扫全量键， + * 免费版很快就会耗尽（issue #125）。索引键记录当前存活记录的完整键名， + * 用 1 次 `get` 取代 1 次（可能多页的）`list`。 + * + * 约束： + * - 只含字母/数字/下划线，满足 EdgeOne KV 的键名限制； + * - 不以任何表前缀开头（`settings_` / `users_` …），不会被 `groupKeysByTable` + * 误当成实体记录，也不会被陈旧键清理误删。 + */ +export const KEY_INDEX_KEY = "openlist_key_index" + /** 构造某张表的键名前缀，如 `users_` */ export function tableKeyPrefix(table: string): string { return `${encodeKeyPart(table)}${KEY_SEP}` diff --git a/src/backend/internal/model/store/store.test.ts b/src/backend/internal/model/store/store.test.ts index 3b03ed79..0b2eea03 100644 --- a/src/backend/internal/model/store/store.test.ts +++ b/src/backend/internal/model/store/store.test.ts @@ -5,6 +5,7 @@ import { readDriver, readFormat, getStoreBackend } from "./backend" import { readPersistedSecret, writePersistedSecret } from "./json" import { mapFormat } from "./format/map" import { keyFormat } from "./format/key" +import { KEY_INDEX_KEY } from "./keycodec" import { sqlFormat } from "./format/sql" import { d1Driver } from "./driver/d1" import { @@ -228,7 +229,10 @@ test("schema: SQL table names align with Go GORM naming", () => { // 完整表名 = 前缀 + 复数名 assert.equal(tableSqlName("settings", {}), "x_setting_items") assert.equal(tableSqlName("shares", {}), "x_sharing_dbs") - assert.equal(tableSqlName("settings", { TABLE_PREFIX: "abc_" }), "x_setting_items") + assert.equal( + tableSqlName("settings", { TABLE_PREFIX: "abc_" }), + "x_setting_items", + ) // DDL 里应包含带前缀的复数表名 assert.ok(D1_SCHEMA.some((d) => d.includes("x_setting_items"))) @@ -289,9 +293,10 @@ test("key format: roundtrip via mock KV driver (per-table keys)", async () => { assert.deepEqual(await keyFormat.load(driver), SAMPLE_DB) }) -test("key format: load/save issue one global list each (issue #125 quota)", async () => { - // 回归防护：key 格式有 6 张表，旧实现 load/save 各逐表 list（6 次）。 - // Cloudflare 免费版 list 配额仅 1000 次/天，必须收敛为每次 1 次全局 list。 +test("key format: global index replaces list() in steady state (issue #125 quota)", async () => { + // 回归防护：Cloudflare 免费版 list 配额仅 1000 次/天。旧实现 load/save + // 各要一次全局 list，正常使用约 1000 次请求就见底。现在用索引键记录存活 + // 记录，稳态下 load/save 都不再调用 list —— 只有首次（索引尚不存在）会扫一次。 const base = createMockKvDriver() let listCalls = 0 const driver: Driver = { @@ -308,12 +313,149 @@ test("key format: load/save issue one global list each (issue #125 quota)", asyn health: base.health, } + // 首次保存：索引不存在，允许 1 次 list 建立索引 + assert.equal(await keyFormat.save(SAMPLE_DB, driver), true) + assert.equal(listCalls, 1) + + // 之后 load/save 完全走索引：list 次数不再增长 + assert.deepEqual(await keyFormat.load(driver), SAMPLE_DB) + assert.equal(listCalls, 1) assert.equal(await keyFormat.save(SAMPLE_DB, driver), true) - assert.equal(listCalls, 1) // save：清空旧数据只 1 次全局 list + assert.equal(listCalls, 1) +}) + +test("key format: legacy layout without index is migrated on first load", async () => { + // 升级前的部署只有实体键 + 初始化标记，没有索引。首次 load 必须仍能读到 + // 全部数据（回退 list），并顺手补写索引，之后不再消耗 list。 + const base = createMockKvDriver() + await keyFormat.save(SAMPLE_DB, base) + await base.delete(KEY_INDEX_KEY) + + let listCalls = 0 + const driver: Driver = { + name: "mock-kv-counting", + isAvailable: base.isAvailable, + init: base.init, + get: base.get, + put: base.put, + delete: base.delete, + list: async (prefix: string) => { + listCalls += 1 + return await base.list(prefix) + }, + health: base.health, + } + + assert.deepEqual(await keyFormat.load(driver), SAMPLE_DB) + assert.equal(listCalls, 1, "首次 load 回退 list 建立索引") + assert.deepEqual(await keyFormat.load(driver), SAMPLE_DB) + assert.equal(listCalls, 1, "索引补写后不再 list") +}) + +test("key format: corrupt index falls back to list and is rewritten", async () => { + const base = createMockKvDriver() + await keyFormat.save(SAMPLE_DB, base) + await base.put(KEY_INDEX_KEY, "{ this is not json") + + let listCalls = 0 + const driver: Driver = { + name: "mock-kv-counting", + isAvailable: base.isAvailable, + init: base.init, + get: base.get, + put: base.put, + delete: base.delete, + list: async (prefix: string) => { + listCalls += 1 + return await base.list(prefix) + }, + health: base.health, + } + + // 索引损坏绝不能当成空库：必须回退 list 并完整读出 + assert.deepEqual(await keyFormat.load(driver), SAMPLE_DB) + assert.equal(listCalls, 1) + assert.deepEqual(await keyFormat.load(driver), SAMPLE_DB) + assert.equal(listCalls, 1, "损坏索引已被重建") +}) + +test("key format: stale records are deleted using the index (no list)", async () => { + const base = createMockKvDriver() + await keyFormat.save(SAMPLE_DB, base) + + let listCalls = 0 + const driver: Driver = { + name: "mock-kv-counting", + isAvailable: base.isAvailable, + init: base.init, + get: base.get, + put: base.put, + delete: base.delete, + list: async (prefix: string) => { + listCalls += 1 + return await base.list(prefix) + }, + health: base.health, + } + + const next = { ...SAMPLE_DB, users: [] } + assert.equal(await keyFormat.save(next, driver), true) + assert.equal(listCalls, 0, "索引已存在时保存不 list") + assert.equal(await base.get("users_1"), null, "被删除的用户键已清理") + + const index = JSON.parse(String(await base.get(KEY_INDEX_KEY))) + assert.ok(!index.keys.includes("users_1")) + assert.deepEqual(await keyFormat.load(driver), next) +}) + +test("key format: records are fetched with bounded concurrency", async () => { + // 串行 get 会让 100+ 条记录的 load 在边缘网络上耗时数秒。这里断言读取确实 + // 并发进行（但不超过上限，避免撞平台子请求限制）。 + const many = { + ...SAMPLE_DB, + users: Array.from({ length: 30 }, (_, i) => ({ + id: i + 1, + username: `u${i + 1}`, + })), + } + const base = createMockKvDriver() + await keyFormat.save(many, base) + + let active = 0 + let maxActive = 0 + const driver: Driver = { + name: "mock-kv-concurrency", + isAvailable: base.isAvailable, + init: base.init, + put: base.put, + delete: base.delete, + list: base.list, + health: base.health, + get: async (key) => { + active += 1 + maxActive = Math.max(maxActive, active) + await new Promise((resolve) => setTimeout(resolve, 2)) + try { + return await base.get(key) + } finally { + active -= 1 + } + }, + } const loaded = await keyFormat.load(driver) - assert.deepEqual(loaded, SAMPLE_DB) - assert.equal(listCalls, 2) // load：再 1 次全局 list（行为与逐表一致） + // 键名是 `users_`，读取顺序与 KV list 一致（字典序），与写入顺序无关； + // 这里只校验全部记录都读到了（并发不能导致漏读/串行解析错误）。 + assert.equal(loaded.users.length, many.users.length) + assert.deepEqual( + loaded.users.map((u: any) => u.id).sort((a: number, b: number) => a - b), + many.users.map((u) => u.id), + ) + assert.ok(maxActive > 1, `expected concurrent reads, saw max=${maxActive}`) + assert.ok( + maxActive <= 12, + `read concurrency must stay bounded, saw ${maxActive}`, + ) }) test("sql format: roundtrip via mock SQL driver (columnar)", async () => { @@ -434,7 +576,9 @@ function createMockD1Binding() { return { results: [] } }, async run() { - if (/INSERT\s+OR\s+REPLACE\s+INTO\s+kv\s*\(key,\s*value\)/i.test(sql)) { + if ( + /INSERT\s+OR\s+REPLACE\s+INTO\s+kv\s*\(key,\s*value\)/i.test(sql) + ) { store.set(String(params[0]), String(params[1])) } else if (/DELETE\s+FROM\s+kv\s+WHERE\s+key\s*=\s*\?/i.test(sql)) { store.delete(String(params[0])) diff --git a/src/backend/internal/op/transfer.ts b/src/backend/internal/op/transfer.ts new file mode 100644 index 00000000..ad4223b1 --- /dev/null +++ b/src/backend/internal/op/transfer.ts @@ -0,0 +1,383 @@ +/** + * 共享传输引擎：把「一个远程 URL」写进某个存储路径。 + * + * 从 `server/seed.ts` 的 uploadStreamToDriver 抽出，供任务系统（跨存储复制/ + * 移动、URL 离线下载、种子转存）复用。抽出时做了两件事： + * 1. 不再依赖 Hono `Context`（后台任务没有请求对象），改用 `TaskRuntime`； + * 2. 分片上传会话与已完成分片由调用方回调持久化，使大文件中途被打断后 + * 能从下一个分片续传，而不是整文件重来。 + * + * 能力优先级（越快越靠前，尽可能让数据不经过 Worker）： + * rapidUpload（秒传）→ putUrl（服务端直传）→ createUploadSession 分片 + * → putStream 流式 → 限内存 buffer + */ +import { resolvePath } from "../model/db" +import { assertSafeUrl, getTrustedHosts } from "../../pkg/http" +import { flushPendingDriverState, getDriver } from "./storage" +import type { StorageRequestContext } from "./storage" + +export interface TaskRuntime { + env?: any + /** 当前部署 origin（用于解析相对 URL / 同源判定） */ + origin?: string + waitUntil?: (promise: Promise) => void + /** + * 大文件中途把断点写回任务 cursor 的回调（由 runner 注入）。 + * 传输引擎只负责在分片完成后回调，不关心任务存储细节。 + */ + checkpoint?: (cursor: unknown) => Promise +} + +export interface TransferSource { + url: string + headers?: Record + size?: number + hashes?: { + md5?: string + sha1?: string + sha256?: string + pieces?: any + [key: string]: any + } +} + +/** 分片上传的断点状态（可 JSON 序列化，字符串 session） */ +export interface MultipartState { + session: string + /** 下一个待上传的分片号（1 起） */ + partNumber: number + uploadedBytes: number + partMd5s: string[] + chunkSize: number + size: number +} + +export interface TransferResult { + method: string + bytes: number + /** 未完成的分片状态；为 null 表示已全部上传 */ + pending?: MultipartState | null + reused?: boolean +} + +export interface TransferOptions { + /** SSRF 白名单（源存储管理员配置的 endpoint + SSRF_ALLOWED_HOSTS） */ + allowHosts?: Set | string[] + maxBufferedBytes?: number + chunkSize?: number + /** 续传状态：由上一次调用通过 onCheckpoint 持久化 */ + resume?: MultipartState | null + /** 每完成一个分片回调一次（调用方负责节流写库） */ + onCheckpoint?: (state: MultipartState) => void | Promise + /** 优先尝试服务端直传（putUrl）。源 URL 需要目标存储可访问 */ + preferServerSide?: boolean +} + +const DEFAULT_CHUNK_SIZE = 8 * 1024 * 1024 +const MAX_CHUNK_SIZE = 16 * 1024 * 1024 +const DEFAULT_BUFFERED_MAX = 4 * 1024 * 1024 +const MAX_REDIRECTS = 4 + +function runtimeContext(rt: TaskRuntime): StorageRequestContext { + return { env: rt.env, waitUntil: rt.waitUntil } +} + +/** + * 逐跳校验 SSRF 的 fetch。 + * + * 不能只校验初始 URL：攻击者可让一个公网 URL 302 到内网/云元数据地址。 + * 因此关闭自动跟随，自己跳转并逐次校验。 + */ +async function safeFetch( + rt: TaskRuntime, + url: string, + init: RequestInit, + allowHosts?: Set | string[], +): Promise { + let current = url + for (let hop = 0; hop <= MAX_REDIRECTS; hop++) { + assertSafeUrl(current, "Task transfer", allowHosts) + const response = await fetch(current, { ...init, redirect: "manual" }) + if (response.status >= 300 && response.status < 400) { + const location = response.headers.get("location") + if (!location) { + throw new Error(`task transfer redirect without location (${response.status})`) + } + current = new URL(location, current).toString() + continue + } + return response + } + throw new Error("task transfer exceeded the redirect limit") +} + +function isReuse(result: any): boolean { + return Boolean(result === true || result?.reuse || result?.success) +} + +/** + * 后台执行没有请求对象：驱动返回的相对 raw_url（如 `/d/...`）必须靠 + * 入队时快照的 origin 或 PUBLIC_ORIGIN 补全，否则会抛出一个难以理解的 + * "Invalid URL"。这里显式给出可操作的错误。 + */ +function absoluteUrl(url: string, origin?: string): string { + if (/^https?:\/\//i.test(url)) return url + if (!origin) { + throw new Error( + "cannot resolve a relative source URL: the task runtime has no deployment origin. " + + "Set PUBLIC_ORIGIN, or re-enqueue the task from an HTTP request.", + ) + } + return new URL(url, origin).toString() +} + +/** + * 把源写入目标路径。失败时抛出可读错误，由 runner 记录并重试。 + */ +export async function transferToPath( + rt: TaskRuntime, + source: TransferSource, + targetVirtualPath: string, + options: TransferOptions = {}, +): Promise { + const resolved = await resolvePath(targetVirtualPath, rt.env) + if (resolved.isVirtual || !resolved.storage) { + throw new Error("target storage was not found") + } + const storage = resolved.storage + const driver = await getDriver(storage.driver, storage) + const dynamic = driver as any + const context = runtimeContext(rt) + const sourceUrl = absoluteUrl(source.url, rt.origin) + const chunkSize = Math.min( + MAX_CHUNK_SIZE, + Math.max(1024 * 1024, options.chunkSize || DEFAULT_CHUNK_SIZE), + ) + const bufferedMax = Math.max( + 0, + options.maxBufferedBytes ?? DEFAULT_BUFFERED_MAX, + ) + + const metadata = { + size: Number(source.size) || 0, + md5: source.hashes?.md5 || "", + sha1: source.hashes?.sha1 || "", + sha256: source.hashes?.sha256 || "", + hashes: source.hashes, + } + + let lastError: any = null + try { + // ── 0. 续传：跳过所有「不需要再碰目标存储」的捷径 ── + const resume = options.resume || null + + // ── 1. 秒传 ── + if (!resume && typeof dynamic.rapidUpload === "function") { + try { + const result = await dynamic.rapidUpload( + targetVirtualPath, + resolved.physical, + metadata, + ) + if (isReuse(result)) { + return { method: "rapidUpload", bytes: metadata.size, reused: true } + } + } catch (err) { + lastError = err + } + } + + // ── 2. 服务端直传（数据不过 Worker，最快） ── + if (!resume && options.preferServerSide && typeof dynamic.putUrl === "function") { + try { + const result = await dynamic.putUrl( + targetVirtualPath, + resolved.physical, + sourceUrl, + metadata, + ) + if (isReuse(result)) { + return { method: "putUrl", bytes: metadata.size } + } + } catch (err) { + lastError = err + } + } + + // ── 3. 分片上传（可续传） ── + let multipart: MultipartState | null = resume + if (!multipart && typeof dynamic.createUploadSession === "function") { + const dir = targetVirtualPath.replace(/\/[^/]*$/, "") || "/" + const name = targetVirtualPath.split("/").filter(Boolean).pop() || "" + const dirResolved = await resolvePath(dir, rt.env) + const session = await dynamic.createUploadSession( + dir, + dirResolved.isVirtual ? "/" : dirResolved.physical, + name, + metadata.size, + metadata.md5, + ) + if (session?.reuse) { + return { method: "multipart-rapid", bytes: metadata.size, reused: true } + } + if (session?.requiresMd5) { + throw new Error( + "target driver requires an MD5 for multipart upload, but the source did not provide one", + ) + } + if (session?.session) { + multipart = { + session: String(session.session), + partNumber: 1, + uploadedBytes: 0, + partMd5s: [], + chunkSize: Number(session.chunkSize) || chunkSize, + size: metadata.size, + } + } + } + + if ( + multipart && + typeof dynamic.uploadPart === "function" && + typeof dynamic.completeUploadSession === "function" + ) { + const state: MultipartState = { ...multipart } + const headers: Record = { ...(source.headers || {}) } + if (state.uploadedBytes > 0) { + headers["Range"] = `bytes=${state.uploadedBytes}-` + } + const response = await safeFetch( + rt, + sourceUrl, + { headers }, + options.allowHosts, + ) + if (!response.ok || !response.body) { + throw new Error(`source download failed with HTTP ${response.status}`) + } + const resumedFromRange = response.status === 206 + const reader = response.body.getReader() + // 源端不支持 Range 时只能整段下载再丢弃已上传字节 + let skip = + state.uploadedBytes > 0 && !resumedFromRange ? state.uploadedBytes : 0 + let buffer = new Uint8Array(0) + let bytes = state.uploadedBytes + const stateChunk = Math.max(1024 * 1024, state.chunkSize || chunkSize) + let lastCheckpoint = Date.now() + try { + while (true) { + const { done, value } = await reader.read() + if (value?.length) { + let chunk = value + if (skip > 0) { + if (chunk.length <= skip) { + skip -= chunk.length + chunk = new Uint8Array(0) + } else { + chunk = chunk.subarray(skip) + skip = 0 + } + } + if (chunk.length) { + const combined = new Uint8Array(buffer.length + chunk.length) + combined.set(buffer) + combined.set(chunk, buffer.length) + buffer = combined + while (buffer.length >= stateChunk) { + const part = buffer.slice(0, stateChunk) + buffer = buffer.slice(stateChunk) + const result = await dynamic.uploadPart( + state.session, + state.partNumber, + part, + ) + if (result?.partMd5) state.partMd5s.push(result.partMd5) + state.partNumber += 1 + bytes += part.length + state.uploadedBytes = bytes + const now = Date.now() + if (options.onCheckpoint && now - lastCheckpoint > 3000) { + lastCheckpoint = now + await options.onCheckpoint({ ...state }) + } + } + } + } + if (done) break + } + if (buffer.length || state.size === 0) { + const result = await dynamic.uploadPart( + state.session, + state.partNumber, + buffer, + ) + if (result?.partMd5) state.partMd5s.push(result.partMd5) + state.partNumber += 1 + bytes += buffer.length + state.uploadedBytes = bytes + } + await dynamic.completeUploadSession(state.session, state.partMd5s) + return { method: "multipart", bytes, pending: null } + } catch (err: any) { + // 把断点挂到错误上：调用方写入 cursor 后，下一次从这里续传。 + // 不把「失败」伪装成「成功 + pending」，否则会掩盖真实错误。 + err.pending = state + throw err + } finally { + try { + reader.releaseLock() + } catch {} + } + } + + // ── 4. 流式上传 ── + const response = await safeFetch(rt, sourceUrl, { headers: source.headers || {} }, options.allowHosts) + if (!response.ok || !response.body) { + throw new Error(`source download failed with HTTP ${response.status}`) + } + if (typeof dynamic.putStream === "function") { + await dynamic.putStream( + targetVirtualPath, + resolved.physical, + response.body, + metadata.size, + metadata, + ) + return { method: "putStream", bytes: metadata.size } + } + + // ── 5. 限内存缓冲 ── + const declared = metadata.size + if (declared > 0 && declared > bufferedMax) { + throw new Error( + `capability unavailable: target driver has no streaming upload API and the file (${declared} bytes) exceeds the ${bufferedMax}-byte buffer limit`, + ) + } + const bytes = new Uint8Array(await response.arrayBuffer()) + if (bytes.byteLength > bufferedMax) { + throw new Error( + `capability unavailable: downloaded ${bytes.byteLength} bytes which exceeds the buffer limit`, + ) + } + await driver.put(targetVirtualPath, resolved.physical!, bytes) + return { method: "buffered", bytes: bytes.byteLength } + } catch (err) { + if (lastError) { + console.warn("[Task transfer] fallback path failed:", lastError) + } + throw err + } finally { + await flushPendingDriverState( + storage.driver, + storage, + driver, + context, + ) + } +} + +/** 源存储的受信 host（管理员配置的内网 endpoint） */ +export function trustedHostsFor(storage: any, env?: any): Set { + return getTrustedHosts(storage?.addition, env) +} diff --git a/src/backend/internal/task/decide.ts b/src/backend/internal/task/decide.ts new file mode 100644 index 00000000..7f36f324 --- /dev/null +++ b/src/backend/internal/task/decide.ts @@ -0,0 +1,132 @@ +/** + * sync / async 判定。 + * + * 用户要求「小于多少 M 的文件同步，否则异步，且阈值可在设置里改」。 + * 判定过程本身也有成本（要探测目录大小），因此带时间与条目上限： + * 探测超限就直接异步，绝不为了「判断快慢」把请求拖死。 + */ +import { resolvePath } from "../model/db" +import { getItem, listItems } from "../op/storage" +import type { TaskRuntime } from "../op/transfer" +import { getTaskSettings } from "./settings" + +export interface TaskModeDecision { + mode: "sync" | "async" + totalBytes: number + entries: number + /** 触发异步的原因，便于日志与前端提示 */ + reason: string +} + +export interface DecisionInput { + op: "copy" | "move" + /** 已套用 base_path 的实际路径 */ + srcDir: string + dstDir: string + names: string[] +} + +function joinPath(dir: string, name: string): string { + return `${dir.replace(/\/+$/, "")}/${name}` +} + +async function resolveSameStorage( + srcDir: string, + dstDir: string, + rt: TaskRuntime, +): Promise { + try { + const src = await resolvePath(srcDir, rt.env) + const dst = await resolvePath(dstDir, rt.env) + if (src.isVirtual || dst.isVirtual) return false + return String(src.storage?.id) === String(dst.storage?.id) + } catch { + return false + } +} + +/** + * 探测总大小。返回 async 时 totalBytes 只是「已发现的部分」，仅用于展示。 + */ +export async function decideCopyMoveMode( + rt: TaskRuntime, + input: DecisionInput, +): Promise { + const settings = await getTaskSettings() + if (!settings.enabled) { + return { mode: "sync", totalBytes: 0, entries: 0, reason: "task system disabled" } + } + if (input.names.length === 0) { + return { mode: "sync", totalBytes: 0, entries: 0, reason: "no names" } + } + + // 同存储的 move 是元数据操作（重命名），与文件大小无关 + if (input.op === "move" && (await resolveSameStorage(input.srcDir, input.dstDir, rt))) { + return { + mode: "sync", + totalBytes: 0, + entries: 0, + reason: "same-storage move (rename)", + } + } + + const context = { env: rt.env, waitUntil: rt.waitUntil } + const deadline = Date.now() + 1500 + const maxEntries = settings.probeMaxEntries + const queue: string[] = input.names.map((name) => joinPath(input.srcDir, name)) + let entries = 0 + let totalBytes = 0 + + while (queue.length > 0) { + if (Date.now() > deadline) { + return { + mode: "async", + totalBytes, + entries, + reason: "size probe exceeded its time budget", + } + } + if (entries >= maxEntries) { + return { + mode: "async", + totalBytes, + entries, + reason: `size probe exceeded ${maxEntries} entries`, + } + } + const path = queue.shift() as string + const { item } = await getItem(path, context) + entries += 1 + if (item.is_dir) { + const listing = await listItems(path, context) + for (const child of listing.content || []) { + queue.push(joinPath(path, child.name)) + } + continue + } + totalBytes += Number(item.size) || 0 + if (totalBytes > settings.syncThresholdBytes) { + return { + mode: "async", + totalBytes, + entries, + reason: `size exceeds ${Math.round(settings.syncThresholdBytes / 1024 / 1024)}MB threshold`, + } + } + } + + if (totalBytes > settings.syncThresholdBytes) { + return { + mode: "async", + totalBytes, + entries, + reason: "size exceeds the configured threshold", + } + } + return { + mode: "sync", + totalBytes, + entries, + reason: `within ${Math.round(settings.syncThresholdBytes / 1024 / 1024)}MB threshold`, + } +} diff --git a/src/backend/internal/task/handlers/copy_move.ts b/src/backend/internal/task/handlers/copy_move.ts new file mode 100644 index 00000000..278c31cf --- /dev/null +++ b/src/backend/internal/task/handlers/copy_move.ts @@ -0,0 +1,495 @@ +/** + * 复制 / 移动任务。 + * + * 单元粒度：一个文件（目录会被懒展开成子项，逐层压栈）。 + * - 同存储：走驱动原生 copy/move（最快）； + * - 跨存储：走共享传输引擎（raw_url → 目标驱动），必要时分片续传。 + * + * 目录按层展开而不是一次遍历整棵树，原因： + * 1. 大目录首次遍历就可能超时； + * 2. 只把待处理项存进 cursor，记录不会随目录规模膨胀。 + */ +import { getDb, resolvePath } from "../../model/db" +import { canCopy, canMove, isWithinUserRoot } from "../../../pkg/permission" +import { + copyItems, + flushPendingDriverState, + getDriver, + getItem, + listItems, + makeDirectory, + moveItems, + removeItems, +} from "../../op/storage" +import { transferToPath, trustedHostsFor } from "../../op/transfer" +import type { MultipartState, TaskRuntime } from "../../op/transfer" +import type { TaskHandler, TaskUnitResult, TaskVerifyResult } from "../runner" +import type { TaskRecord } from "../types" + +export type CopyMoveOp = "copy" | "move" + +export interface CopyMovePayload { + op: CopyMoveOp + /** 已套用用户 base_path 的实际路径 */ + src_dir: string + dst_dir: string + names: string[] + /** 入队时的部署 origin：后台没有请求对象，相对 raw_url 需要它才能解析 */ + origin?: string +} + +interface StackEntry { + src: string + dst: string + dir: boolean + size: number +} + +interface CopyMoveCursor { + /** 下一个顶层条目下标 */ + index: number + stack: StackEntry[] + totalBytes: number + doneBytes: number + files: number + doneFiles: number + dirs: number + doneDirs: number + /** 是否同存储（懒计算，避免每个 unit 都解析两次） */ + sameStorage?: boolean + /** 当前正在传输的文件（配合 upload 断点） */ + current?: string | null + upload?: MultipartState | null + /** 移动任务用：处理完后尝试删除已空的源目录 */ + dirsToRemove?: string[] + cleaned?: boolean +} + +const CTX = (rt: TaskRuntime) => ({ env: rt.env, waitUntil: rt.waitUntil }) + +function trimEnd(value: string): string { + return value.replace(/\/+$/, "") +} + +function joinPath(dir: string, name: string): string { + return `${trimEnd(dir)}/${name}` +} + +function parentOf(path: string): string { + const trimmed = trimEnd(path) + const index = trimmed.lastIndexOf("/") + return index <= 0 ? "/" : trimmed.slice(0, index) +} + +function baseName(path: string): string { + const parts = trimEnd(path).split("/").filter(Boolean) + return parts.pop() || "" +} + +function emptyCursor(): CopyMoveCursor { + return { + index: 0, + stack: [], + totalBytes: 0, + doneBytes: 0, + files: 0, + doneFiles: 0, + dirs: 0, + doneDirs: 0, + current: null, + upload: null, + dirsToRemove: [], + cleaned: false, + } +} + +function readCursor(value: unknown): CopyMoveCursor { + const cursor = (value || {}) as Partial + return { + ...emptyCursor(), + ...cursor, + stack: Array.isArray(cursor.stack) ? cursor.stack : [], + dirsToRemove: Array.isArray(cursor.dirsToRemove) ? cursor.dirsToRemove : [], + } +} + +function readPayload(value: unknown): CopyMovePayload { + const payload = (value || {}) as Partial + if (!Array.isArray(payload.names) || payload.names.length === 0) { + throw new Error("invalid task payload: names is empty") + } + return { + op: payload.op === "move" ? "move" : "copy", + src_dir: String(payload.src_dir || "/"), + dst_dir: String(payload.dst_dir || "/"), + names: payload.names.map((name) => String(name)), + origin: typeof payload.origin === "string" ? payload.origin : undefined, + } +} + +/** 后台执行时没有请求对象：优先用入队时快照的 origin，其次环境变量 */ +function runtimeWithOrigin( + rt: TaskRuntime, + payload: CopyMovePayload, +): TaskRuntime { + if (rt.origin) return rt + const origin = + payload.origin || + rt.env?.PUBLIC_ORIGIN || + (typeof process !== "undefined" ? process.env?.PUBLIC_ORIGIN : "") || + "" + return origin ? { ...rt, origin: String(origin) } : rt +} + +/** + * 执行前重新校验创建者的权限。 + * + * 入队时校验过，但任务可能在几小时后才执行：期间用户可能被禁用、权限被收回、 + * 根目录被调整。这里必须重新确认，否则等于把旧权限永久固化。 + */ +async function assertTaskUser( + task: TaskRecord, + payload: CopyMovePayload, +): Promise { + const db = await getDb() + const user = (db.users || []).find((u: any) => u.id === task.creator_id) + if (!user) throw new Error("task creator no longer exists") + if (user.disabled) throw new Error("task creator is disabled") + if (payload.op === "copy" && !canCopy(user)) { + throw new Error("task creator no longer has the copy permission") + } + if (payload.op === "move" && !canMove(user)) { + throw new Error("task creator no longer has the move permission") + } + for (const path of [payload.src_dir, payload.dst_dir]) { + if (!isWithinUserRoot(user, path)) { + throw new Error("task paths are outside the creator's root directory") + } + } +} + +/** mkdir 的幂等包装：多数驱动在目录已存在时会抛错，这在断点续跑里是正常情况 */ +async function mkdirSafe(path: string, rt: TaskRuntime): Promise { + try { + await makeDirectory(path, CTX(rt)) + } catch (err: any) { + const message = String(err?.message || err || "") + if (/exist|already/i.test(message)) return + throw err + } +} + +async function sourceLink( + virtualPath: string, + rt: TaskRuntime, +): Promise<{ + url: string + headers?: Record + size: number + hashes?: any + storage: any +}> { + const resolved = await resolvePath(virtualPath, rt.env) + if (resolved.isVirtual || !resolved.storage) { + throw new Error("source storage was not found") + } + const driver = await getDriver(resolved.storage.driver, resolved.storage) + try { + const item = await driver.get(virtualPath, resolved.physical!) + if (!item?.raw_url) { + throw new Error( + `source driver returned no download link${ + item?.raw_url_error ? `: ${item.raw_url_error}` : "" + }`, + ) + } + return { + url: item.raw_url, + headers: item.raw_url_headers, + size: Number(item.size) || 0, + hashes: item.hashes || (item.hash ? { md5: item.hash } : undefined), + storage: resolved.storage, + } + } finally { + await flushPendingDriverState( + resolved.storage.driver, + resolved.storage, + driver, + CTX(rt), + ) + } +} + +async function detectSameStorage( + payload: CopyMovePayload, + rt: TaskRuntime, +): Promise { + try { + const src = await resolvePath(payload.src_dir, rt.env) + const dst = await resolvePath(payload.dst_dir, rt.env) + if (src.isVirtual || dst.isVirtual) return false + return String(src.storage?.id) === String(dst.storage?.id) + } catch { + return false + } +} + +/** 把「下一个顶层名字」变成栈项；返回 false 表示没有更多顶层条目 */ +async function pushTopLevel( + cursor: CopyMoveCursor, + payload: CopyMovePayload, + rt: TaskRuntime, +): Promise { + if (cursor.index >= payload.names.length) return false + const name = payload.names[cursor.index] + cursor.index += 1 + if (!name) return true + const src = joinPath(payload.src_dir, name) + const dst = joinPath(payload.dst_dir, name) + const { item } = await getItem(src, CTX(rt)) + const entry: StackEntry = { + src, + dst, + dir: Boolean(item.is_dir), + size: Number(item.size) || 0, + } + cursor.stack.push(entry) + if (!entry.dir) { + cursor.files += 1 + cursor.totalBytes += entry.size + } + return true +} + +async function runUnitImpl( + rt: TaskRuntime, + task: TaskRecord, +): Promise { + const payload = readPayload(task.payload) + const taskRt = runtimeWithOrigin(rt, payload) + const cursor = readCursor(task.cursor) + await assertTaskUser(task, payload) + + if (cursor.sameStorage === undefined) { + cursor.sameStorage = await detectSameStorage(payload, taskRt) + } + + // 栈空：要么补充下一个顶层条目，要么进入收尾 + if (cursor.stack.length === 0) { + const pushed = await pushTopLevel(cursor, payload, taskRt) + if (pushed) { + return { + cursor, + totalBytes: cursor.totalBytes, + doneBytes: cursor.doneBytes, + status: `scanning ${baseName(cursor.stack[0]?.src || "")}`, + } + } + + // 移动的跨存储场景：文件删完了，再把空的源目录自底向上删掉 + if (payload.op === "move" && !cursor.cleaned) { + const dirs = [...(cursor.dirsToRemove || [])].reverse() + for (const dir of dirs) { + try { + await removeItems(parentOf(dir), [baseName(dir)], CTX(taskRt)) + } catch { + // 目录非空或驱动不支持删除：忽略，源端最多留空目录 + } + } + cursor.cleaned = true + cursor.dirsToRemove = [] + return { + cursor, + totalBytes: cursor.totalBytes, + doneBytes: cursor.doneBytes, + status: "cleaning empty source directories", + } + } + + return { + cursor, + totalBytes: cursor.totalBytes, + doneBytes: cursor.doneBytes, + finished: true, + status: `${payload.op} completed`, + } + } + + const entry = cursor.stack.pop() as StackEntry + + if (entry.dir) { + await mkdirSafe(entry.dst, taskRt) + if (payload.op === "move") { + cursor.dirsToRemove = [...(cursor.dirsToRemove || []), entry.src] + } + const listing = await listItems(entry.src, CTX(taskRt)) + const children = [...(listing.content || [])].sort((a, b) => + a.name.localeCompare(b.name), + ) + // 逆序压栈，保证按名字顺序处理（栈是后进先出） + for (let i = children.length - 1; i >= 0; i--) { + const child = children[i] + cursor.stack.push({ + src: joinPath(entry.src, child.name), + dst: joinPath(entry.dst, child.name), + dir: Boolean(child.is_dir), + size: Number(child.size) || 0, + }) + if (child.is_dir) { + cursor.dirs += 1 + } else { + cursor.files += 1 + cursor.totalBytes += Number(child.size) || 0 + } + } + cursor.doneDirs += 1 + return { + cursor, + totalBytes: cursor.totalBytes, + doneBytes: cursor.doneBytes, + status: `${payload.op} ${baseName(entry.src)}/`, + } + } + + // ── 文件：同存储走驱动原生接口 ── + if (cursor.sameStorage) { + if (payload.op === "move") { + await moveItems( + parentOf(entry.src), + parentOf(entry.dst), + [baseName(entry.src)], + CTX(taskRt), + ) + } else { + await copyItems( + parentOf(entry.src), + parentOf(entry.dst), + [baseName(entry.src)], + CTX(taskRt), + ) + } + cursor.doneFiles += 1 + cursor.doneBytes += entry.size + cursor.current = null + cursor.upload = null + return { + cursor, + totalBytes: cursor.totalBytes, + doneBytes: cursor.doneBytes, + status: `${payload.op} ${baseName(entry.src)}`, + } + } + + // ── 文件：跨存储 ── + // 先记录当前文件，断点才可能命中；只在切换文件时写一次 + if (cursor.current !== entry.src) { + cursor.current = entry.src + cursor.upload = null + await rt.checkpoint?.(cursor) + } + + const source = await sourceLink(entry.src, taskRt) + const result = await transferToPath( + taskRt, + { + url: source.url, + headers: source.headers, + size: source.size || entry.size, + hashes: source.hashes, + }, + entry.dst, + { + allowHosts: trustedHostsFor(source.storage, taskRt.env), + preferServerSide: true, + resume: cursor.upload || null, + onCheckpoint: async (state) => { + cursor.upload = state + await rt.checkpoint?.(cursor) + }, + }, + ) + + cursor.doneFiles += 1 + cursor.doneBytes += Math.max(entry.size, result.bytes) + cursor.current = null + cursor.upload = null + + // 移动：传输成功后再删源，避免中途失败导致数据丢失 + if (payload.op === "move") { + await removeItems(parentOf(entry.src), [baseName(entry.src)], CTX(taskRt)) + } + + return { + cursor, + totalBytes: cursor.totalBytes, + doneBytes: cursor.doneBytes, + status: `${payload.op} ${baseName(entry.src)}`, + } +} + +async function verifyImpl( + rt: TaskRuntime, + task: TaskRecord, +): Promise { + const payload = readPayload(task.payload) + const cursor = readCursor(task.cursor) + const taskRt = runtimeWithOrigin(rt, payload) + + // 按父目录缓存列表结果：多个顶层条目通常落在同一个目录下 + const listCache = new Map>() + const listNames = async (dir: string): Promise> => { + const cached = listCache.get(dir) + if (cached) return cached + const listing = await listItems(dir, CTX(taskRt)) + const set = new Set((listing.content || []).map((item) => item.name)) + listCache.set(dir, set) + return set + } + + const problems: string[] = [] + for (const name of payload.names) { + const dstPath = joinPath(payload.dst_dir, name) + const dstNames = await listNames(parentOf(dstPath)) + if (!dstNames.has(name)) { + problems.push(`missing target: ${dstPath}`) + continue + } + if (payload.op === "move") { + const srcPath = joinPath(payload.src_dir, name) + const srcNames = await listNames(parentOf(srcPath)) + if (srcNames.has(name)) problems.push(`source still exists: ${srcPath}`) + } + } + + if (problems.length > 0) { + return { + ok: false, + error: `${problems.length} verification problem(s): ${problems + .slice(0, 20) + .join(", ")}`, + } + } + if (cursor.files > 0 && cursor.doneFiles < cursor.files) { + return { + ok: false, + error: `only ${cursor.doneFiles}/${cursor.files} file(s) were transferred`, + } + } + return { ok: true } +} + +const impl = { + async runUnit( + rt: TaskRuntime, + task: TaskRecord, + _deadline: number, + ): Promise { + return runUnitImpl(rt, task) + }, + async verify(rt: TaskRuntime, task: TaskRecord): Promise { + return verifyImpl(rt, task) + }, +} + +export const copyTaskHandler: TaskHandler = { type: "copy", ...impl } +export const moveTaskHandler: TaskHandler = { type: "move", ...impl } diff --git a/src/backend/internal/task/handlers/offline_download.test.ts b/src/backend/internal/task/handlers/offline_download.test.ts new file mode 100644 index 00000000..df27e945 --- /dev/null +++ b/src/backend/internal/task/handlers/offline_download.test.ts @@ -0,0 +1,68 @@ +import assert from "node:assert/strict" +import { test } from "node:test" + +import { + assertDownloadUrlAllowed, + fileNameFromUrl, + hostAllowed, +} from "./offline_download" + +test("offline download is disabled unless a host allowlist is configured", () => { + assert.throws( + () => assertDownloadUrlAllowed("https://example.com/a.zip", {}), + /disabled/i, + ) +}) + +test("non-allowlisted hosts are rejected", () => { + assert.throws( + () => + assertDownloadUrlAllowed("https://evil.example/a.zip", { + ALLOW_OFFLINE_DOWNLOAD: "example.com", + }), + /not in the ALLOW_OFFLINE_DOWNLOAD allowlist/, + ) +}) + +test("allowlisted hosts are accepted", () => { + assert.doesNotThrow(() => + assertDownloadUrlAllowed("https://example.com/a.zip", { + ALLOW_OFFLINE_DOWNLOAD: "example.com", + }), + ) +}) + +test("wildcard entries match subdomains but not the bare domain", () => { + assert.equal(hostAllowed("a.example.com", ["*.example.com"]), true) + assert.equal(hostAllowed("example.com", ["*.example.com"]), false) + assert.equal(hostAllowed("example.com", ["example.com"]), true) +}) + +test("private network targets stay blocked even when allowlisted", () => { + assert.throws( + () => + assertDownloadUrlAllowed("http://127.0.0.1/a.zip", { + ALLOW_OFFLINE_DOWNLOAD: "127.0.0.1", + }), + /SSRF|restricted|private/i, + ) +}) + +test("only http(s) URLs are accepted", () => { + assert.throws( + () => + assertDownloadUrlAllowed("file:///etc/passwd", { + ALLOW_OFFLINE_DOWNLOAD: "example.com", + }), + /http\(s\)/i, + ) +}) + +test("file names are derived from the URL and sanitized", () => { + assert.equal(fileNameFromUrl("https://example.com/dir/movie.mkv"), "movie.mkv") + assert.equal( + fileNameFromUrl("https://example.com/dir/a%3Ab:c.zip"), + "a_b_c.zip", + ) + assert.match(fileNameFromUrl("https://example.com/"), /^download-\d+$/) +}) diff --git a/src/backend/internal/task/handlers/offline_download.ts b/src/backend/internal/task/handlers/offline_download.ts new file mode 100644 index 00000000..a3b2a14c --- /dev/null +++ b/src/backend/internal/task/handlers/offline_download.ts @@ -0,0 +1,192 @@ +/** + * URL 离线下载任务。 + * + * 安全模型（默认关闭）： + * - 必须显式配置 `ALLOW_OFFLINE_DOWNLOAD`（主机白名单，逗号/空格分隔， + * 支持 `*.example.com`）；空表示拒绝一切外网下载，避免被当成免费代理 + * 刷流量。 + * - 仍然走 `assertSafeUrl` 的 SSRF 防护（私网/云元数据地址一律拒绝）。 + * - 入队时校验一次，执行时再校验一次（配置可能在任务排队期间被收紧）。 + */ +import { assertSafeUrl } from "../../../pkg/http" +import { getItem, listItems } from "../../op/storage" +import { transferToPath } from "../../op/transfer" +import type { TaskRuntime } from "../../op/transfer" +import type { TaskHandler, TaskUnitResult, TaskVerifyResult } from "../runner" +import type { TaskRecord } from "../types" + +export interface OfflineDownloadPayload { + url: string + /** 已套用 base_path 的目标目录 */ + dst_dir: string + /** 目标文件名（入队时已清洗） */ + name: string + origin?: string +} + +/** 读取主机白名单：`ALLOW_OFFLINE_DOWNLOAD`（也兼容 `OFFLINE_DOWNLOAD_ALLOW`） */ +export function offlineDownloadAllowList(env?: any): string[] { + const raw = + env?.ALLOW_OFFLINE_DOWNLOAD ?? + env?.OFFLINE_DOWNLOAD_ALLOW ?? + (typeof process !== "undefined" + ? (process.env?.ALLOW_OFFLINE_DOWNLOAD ?? + process.env?.OFFLINE_DOWNLOAD_ALLOW) + : "") + return String(raw || "") + .split(/[\s,;]+/) + .map((host) => host.trim().toLowerCase()) + .filter(Boolean) +} + +export function hostAllowed(hostname: string, allowlist: string[]): boolean { + const host = hostname.toLowerCase() + return allowlist.some((entry) => + entry.startsWith("*.") + ? host.endsWith(entry.slice(1)) && host !== entry.slice(2) + : host === entry, + ) +} + +/** 校验一个下载地址：协议、SSRF、主机白名单 */ +export function assertDownloadUrlAllowed(url: string, env?: any): void { + const allowlist = offlineDownloadAllowList(env) + if (allowlist.length === 0) { + throw new Error( + "offline download is disabled: set ALLOW_OFFLINE_DOWNLOAD to a comma-separated host allowlist", + ) + } + let parsed: URL + try { + parsed = new URL(url) + } catch { + throw new Error(`invalid URL: ${url}`) + } + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { + throw new Error("only http(s) URLs are supported") + } + assertSafeUrl(parsed.toString(), "Offline download") + if (!hostAllowed(parsed.hostname, allowlist)) { + throw new Error( + `host "${parsed.hostname}" is not in the ALLOW_OFFLINE_DOWNLOAD allowlist`, + ) + } +} + +/** 从 URL 推导一个安全的文件名 */ +export function fileNameFromUrl(url: string): string { + try { + const parsed = new URL(url) + const last = parsed.pathname.split("/").filter(Boolean).pop() || "" + const decoded = decodeURIComponent(last) + const clean = decoded.replace(/[\\/:*?"<>|\u0000-\u001f]/g, "_").trim() + if (clean && clean !== "." && clean !== "..") return clean + } catch { + // 落到默认名 + } + return `download-${Date.now()}` +} + +function readPayload(value: unknown): OfflineDownloadPayload { + const payload = (value || {}) as Partial + if (!payload.url) throw new Error("invalid task payload: url is missing") + if (!payload.dst_dir) throw new Error("invalid task payload: dst_dir is missing") + return { + url: String(payload.url), + dst_dir: String(payload.dst_dir), + name: String(payload.name || fileNameFromUrl(String(payload.url))), + origin: typeof payload.origin === "string" ? payload.origin : undefined, + } +} + +function withOrigin( + rt: TaskRuntime, + payload: OfflineDownloadPayload, +): TaskRuntime { + if (rt.origin) return rt + const origin = payload.origin || rt.env?.PUBLIC_ORIGIN || "" + return origin ? { ...rt, origin: String(origin) } : rt +} + +function joinPath(dir: string, name: string): string { + return `${dir.replace(/\/+$/, "")}/${name}` +} + +function targetPathOf(payload: OfflineDownloadPayload): string { + return joinPath(payload.dst_dir, payload.name) +} + +async function runUnitImpl( + rt: TaskRuntime, + task: TaskRecord, +): Promise { + const payload = readPayload(task.payload) + const taskRt = withOrigin(rt, payload) + // 执行时重新校验：白名单可能在任务排队期间被收紧 + assertDownloadUrlAllowed(payload.url, taskRt.env) + + const cap = + Number(taskRt.env?.TASK_DOWNLOAD_MAX_BYTES) > 0 + ? Number(taskRt.env?.TASK_DOWNLOAD_MAX_BYTES) + : 64 * 1024 * 1024 + + const result = await transferToPath( + taskRt, + { url: payload.url }, + targetPathOf(payload), + { + maxBufferedBytes: cap, + preferServerSide: false, + }, + ) + + return { + finished: true, + totalBytes: result.bytes, + doneBytes: result.bytes, + cursor: { downloaded: result.bytes, method: result.method }, + status: `downloaded via ${result.method}`, + } +} + +async function verifyImpl( + rt: TaskRuntime, + task: TaskRecord, +): Promise { + const payload = readPayload(task.payload) + const taskRt = withOrigin(rt, payload) + const target = targetPathOf(payload) + const parent = target.slice(0, target.lastIndexOf("/")) || "/" + try { + const listing = await listItems(parent, { + env: taskRt.env, + waitUntil: taskRt.waitUntil, + }) + const found = (listing.content || []).find( + (item) => item.name === payload.name, + ) + if (!found) return { ok: false, error: `target file not found: ${target}` } + return { ok: true } + } catch (err: any) { + // list 不可用时退化为按路径探测 + try { + await getItem(target, { env: taskRt.env, waitUntil: taskRt.waitUntil }) + return { ok: true } + } catch { + return { + ok: false, + error: `failed to verify ${target}: ${err?.message || err}`, + } + } + } +} + +export const offlineDownloadHandler: TaskHandler = { + type: "offline_download", + async runUnit(rt, task) { + return runUnitImpl(rt, task) + }, + async verify(rt, task) { + return verifyImpl(rt, task) + }, +} diff --git a/src/backend/internal/task/manager.ts b/src/backend/internal/task/manager.ts new file mode 100644 index 00000000..c94a6baa --- /dev/null +++ b/src/backend/internal/task/manager.ts @@ -0,0 +1,383 @@ +/** + * 任务管理：入队、查询、取消/重试/删除、配额与清理。 + * + * 这一层同时服务两个入口： + * - `/api/task/*`（前端任务页） + * - `/fs/*`（复制/移动等写操作在判定为 async 后入队） + * + * 存储选择：有 TASK_DO 绑定用 Durable Object，否则用文档存储。 + */ +import { + ACTIVE_STATES, + DONE_STATES, + TASK_STATE, + createTaskRecord, + toTaskInfo, + type TaskInfo, + type TaskRecord, + type TaskType, +} from "./types" +import type { TaskAdmin } from "./store" +import { DocTaskStore } from "./store_doc" +import { DoTaskClient, getTaskDoStub, hasTaskDo } from "./store_do_client" +import { getTaskSettings, resolveTickBudget, type TaskSettings } from "./settings" +import { TASK_LEASE_MS, TASK_STALE_MS, runTick } from "./runner" +import { setEnvCtx } from "../model/db" +import type { TaskRuntime } from "../op/transfer" + +/** 单个用户同时存在的未完成任务上限，防止刷爆存储配额 */ +export const MAX_ACTIVE_TASKS_PER_USER = 100 + +export class TaskError extends Error { + constructor( + message: string, + readonly status: number, + readonly code: string, + ) { + super(message) + this.name = "TaskError" + } +} + +export interface TaskActor { + id: number + username: string + role: number + isAdmin: boolean +} + +export async function getTaskAdmin(env?: any): Promise { + const stub = getTaskDoStub(env) + if (stub) return new DoTaskClient(stub) + return DocTaskStore.create(env) +} + +function assertOwner(record: TaskRecord | null, actor: TaskActor): TaskRecord { + // 与 Go 一致：非本人一律 404，避免用错误码差异探测 tid 是否存在 + if (!record) throw new TaskError("task not found", 404, "NOT_FOUND") + if (!actor.isAdmin && record.creator_id !== actor.id) { + throw new TaskError("task not found", 404, "NOT_FOUND") + } + return record +} + +export interface EnqueueInput { + type: TaskType + name: string + creator_id: number + creator: string + creator_role: number + payload: unknown + cursor?: unknown + total_bytes?: number +} + +/** 入队；调用方负责判定 sync/async 与权限校验（此时已完成） */ +export async function enqueueTask( + env: any, + input: EnqueueInput, +): Promise { + const settings = await getTaskSettings() + if (!settings.enabled) { + throw new TaskError( + "task system is disabled by settings (task_enabled=false)", + 503, + "TASKS_DISABLED", + ) + } + const store = await getTaskAdmin(env) + const active = await store.countActive(input.creator_id) + if (active >= MAX_ACTIVE_TASKS_PER_USER) { + throw new TaskError( + `too many active tasks (${active}); wait for some to finish or clear them`, + 429, + "TASK_LIMIT", + ) + } + const record = createTaskRecord({ + type: input.type, + name: input.name, + creator_id: input.creator_id, + creator: input.creator, + creator_role: input.creator_role, + max_retry: settings.maxRetry[input.type], + payload: input.payload, + cursor: input.cursor, + total_bytes: input.total_bytes, + }) + await store.enqueue(record) + if (store.prune) { + try { + await store.prune(input.type, settings.keepDone) + } catch (err) { + console.warn("[Task] prune failed:", err) + } + } + return record +} + +function stateFilter(which: "undone" | "done"): number[] { + return which === "done" ? DONE_STATES : ACTIVE_STATES +} + +/** + * 记录一条「已在请求内完成」的任务（同步路径）。 + * + * 对齐 Go 的行为：小操作虽然同步完成，也会出现在任务页的 done 列表里。 + * 写失败绝不影响主操作——任务历史是附加信息，不是正确性的一部分。 + */ +export async function recordCompletedTask( + env: any, + input: EnqueueInput, + status = "completed", +): Promise { + try { + const settings = await getTaskSettings() + if (!settings.enabled) return + const store = await getTaskAdmin(env) + const record = createTaskRecord({ + type: input.type, + name: input.name, + creator_id: input.creator_id, + creator: input.creator, + creator_role: input.creator_role, + max_retry: 0, + payload: input.payload, + total_bytes: input.total_bytes, + }) + const now = Date.now() + record.state = TASK_STATE.succeeded + record.progress = 100 + record.done_bytes = record.total_bytes + record.start_time = new Date(now).toISOString() + record.end_time = new Date(now).toISOString() + record.status = status + record.verified = true + record.updated_at = now + await store.enqueue(record) + if (store.prune) { + try { + await store.prune(input.type, settings.keepDone) + } catch {} + } + } catch (err) { + console.warn("[Task] failed to record completed task:", err) + } +} + +export async function listTaskInfos( + env: any, + type: TaskType, + which: "undone" | "done", + actor: TaskActor, +): Promise { + const store = await getTaskAdmin(env) + const records = await store.list({ + type, + states: stateFilter(which), + creator: actor.isAdmin ? undefined : actor.id, + limit: 200, + }) + return records.map(toTaskInfo) +} + +export async function getTaskDetail( + env: any, + type: TaskType, + id: string, + actor: TaskActor, +): Promise { + const store = await getTaskAdmin(env) + return toTaskInfo(assertOwner(await store.get(type, id), actor)) +} + +type TaskAction = "cancel" | "delete" | "retry" + +export async function applyTaskAction( + env: any, + type: TaskType, + id: string, + actor: TaskActor, + action: TaskAction, +): Promise { + const store = await getTaskAdmin(env) + const record = assertOwner(await store.get(type, id), actor) + await finishAction(store, record, action) +} + +async function finishAction( + store: TaskAdmin, + record: TaskRecord, + action: TaskAction, +): Promise { + const now = Date.now() + if (action === "delete") { + // 先置 canceled 再删：runner 下一次读不到记录会自行停止 + if (!DONE_STATES.includes(record.state)) { + await store.patch(record.type, record.id, { + state: TASK_STATE.canceled, + status: "deleted", + updated_at: now, + lease: null, + }) + } + await store.remove(record.type, record.id) + return + } + + if (action === "cancel") { + if (DONE_STATES.includes(record.state)) return + // 正在执行 → canceling，让 runner 在下一个 unit 边界收尾； + // 未执行 → 直接 canceled，不依赖是否有 runner 在跑。 + const running = record.state === TASK_STATE.running + await store.patch(record.type, record.id, { + state: running ? TASK_STATE.canceling : TASK_STATE.canceled, + status: running ? "canceling" : "canceled", + end_time: running ? null : new Date(now).toISOString(), + updated_at: now, + lease: running ? record.lease || null : null, + }) + return + } + + // retry + if (!DONE_STATES.includes(record.state)) { + throw new TaskError( + "task is still active; cancel it before retrying", + 400, + "TASK_ACTIVE", + ) + } + await store.patch(record.type, record.id, { + state: TASK_STATE.pending, + status: "retrying", + retry: 0, + attempts: 0, + error: "", + end_time: null, + next_run_at: now, + updated_at: now, + lease: null, + }) +} + +export interface BulkResult { + [tid: string]: string +} + +export async function applyBulkAction( + env: any, + type: TaskType, + ids: string[], + actor: TaskActor, + action: TaskAction, +): Promise { + const store = await getTaskAdmin(env) + const result: BulkResult = {} + for (const id of ids) { + try { + const record = assertOwner(await store.get(type, id), actor) + await finishAction(store, record, action) + } catch (err: any) { + result[id] = String(err?.message || err) + } + } + return result +} + +export async function clearDoneTasks( + env: any, + type: TaskType, + actor: TaskActor, + onlySucceeded: boolean, +): Promise { + const store = await getTaskAdmin(env) + const states = onlySucceeded ? [TASK_STATE.succeeded] : DONE_STATES + const records = await store.list({ + type, + states, + creator: actor.isAdmin ? undefined : actor.id, + limit: 500, + }) + await store.removeMany(records.map((r) => ({ type: r.type, id: r.id }))) +} + +export async function retryFailedTasks( + env: any, + type: TaskType, + actor: TaskActor, +): Promise { + const store = await getTaskAdmin(env) + const records = await store.list({ + type, + states: [TASK_STATE.failed], + creator: actor.isAdmin ? undefined : actor.id, + limit: 200, + }) + for (const record of records) { + await finishAction(store, record, "retry") + } +} + +/** + * 入队后立即启动执行。 + * + * - 有 DO：RPC 让 DO 自己排 alarm（入队请求返回后任务马上开始）； + * - 无 DO：用 waitUntil 跑一个内联时间片（EdgeOne/Node 等）。 + */ +export async function kickTaskPump( + env: any, + runtime: TaskRuntime = {}, +): Promise { + const stub = getTaskDoStub(env) + if (stub) { + const promise = Promise.resolve(stub.tick({})) + if (runtime.waitUntil) runtime.waitUntil(promise) + else await promise + return + } + const settings = await getTaskSettings() + if (!settings.enabled) return + const store = await DocTaskStore.create(env) + const tick = runTick(store, { ...runtime, env }, { + budgetMs: resolveTickBudget(settings, "inline"), + concurrency: 1, + owner: `inline:${Date.now()}`, + }) + if (runtime.waitUntil) runtime.waitUntil(tick) + else await tick +} + +/** `/api/task/run` 使用：同步跑一个 tick 并返回结果 */ +export async function runTaskPump( + env: any, + runtime: TaskRuntime, + options: { budgetMs?: number; pump?: "http" | "cron" | "esa" | "node" } = {}, +): Promise<{ ran: number; remaining: number; progressed: boolean; driver: string }> { + setEnvCtx(env) + const settings = await getTaskSettings() + const budgetMs = + options.budgetMs && options.budgetMs > 0 + ? Math.min(60000, Math.max(200, options.budgetMs)) + : resolveTickBudget(settings, options.pump || "http") + const stub = getTaskDoStub(env) + if (stub) { + const result = await stub.tick({ budgetMs }) + return { ...result, driver: "do" } + } + const store = await DocTaskStore.create(env) + const result = await runTick(store, runtime, { + budgetMs, + concurrency: settings.parallelUnits, + owner: `http:${Date.now()}`, + }) + return { ...result, driver: "doc" } +} + +/** 是否存在 DO（用于诊断与前端提示） */ +export function taskStoreKind(env?: any): "do" | "doc" { + return hasTaskDo(env) ? "do" : "doc" +} + +export { TASK_LEASE_MS, TASK_STALE_MS } +export type { TaskSettings } diff --git a/src/backend/internal/task/runner.ts b/src/backend/internal/task/runner.ts new file mode 100644 index 00000000..22836eed --- /dev/null +++ b/src/backend/internal/task/runner.ts @@ -0,0 +1,363 @@ +/** + * 任务执行器：分片推进 + 断点续跑 + 取消/重试 + 完成前校验。 + * + * 设计约束： + * - 一次 tick 有硬时间预算（默认 2s，可配），预算内可执行多个 unit； + * - 每个 unit 完成后只写「变化的列」，避免整条记录重写（DO 按行计费）； + * - 崩溃恢复：running 且超时未见更新的任务会被重新认领（见各 store 的 claim）； + * - 连续无进展会被熔断，避免无限重试烧配额。 + */ +import { getTaskSettings, type TaskSettings } from "./settings" +import type { TaskRunnerStore } from "./store" +import { + TASK_STATE, + computeProgress, + isTerminalState, + RUNNABLE_TYPES, + type TaskRecord, + type TaskType, +} from "./types" +import type { TaskRuntime } from "../op/transfer" +import { copyTaskHandler, moveTaskHandler } from "./handlers/copy_move" +import { offlineDownloadHandler } from "./handlers/offline_download" + +export interface TaskUnitResult { + cursor?: unknown + totalBytes?: number + doneBytes?: number + progress?: number + status?: string + /** 所有 unit 已完成（runner 会接着跑 verify） */ + finished?: boolean +} + +export interface TaskVerifyResult { + ok: boolean + error?: string +} + +export interface TaskHandler { + type: TaskType + /** 执行一个最小单元（一个文件 / 一个目录 / 一个条目 / 一个 URL） */ + runUnit( + rt: TaskRuntime, + task: TaskRecord, + deadline: number, + ): Promise + /** 置为 succeeded 之前的一致性校验 */ + verify(rt: TaskRuntime, task: TaskRecord): Promise +} + +const HANDLERS: Partial> = { + copy: copyTaskHandler, + move: moveTaskHandler, + offline_download: offlineDownloadHandler, +} + +/** 仅供测试：注入/移除 handler（生产代码不要调用） */ +export function __setTaskHandlerForTest( + type: TaskType, + handler: TaskHandler | null, +): void { + if (handler) HANDLERS[type] = handler + else delete HANDLERS[type] +} + +export function getHandler(type: TaskType): TaskHandler | undefined { + return HANDLERS[type] +} + +export function hasHandler(type: TaskType): boolean { + return Boolean(HANDLERS[type]) +} + +/** 进度落盘最小间隔：过于频繁会打满 DO 的「行写」/ KV 的写配额 */ +const PROGRESS_FLUSH_MS = 3000 +/** 取消检查间隔 */ +const CANCEL_CHECK_MS = 3000 +/** running 超过该时长没更新，视为僵尸任务 */ +export const TASK_STALE_MS = 5 * 60 * 1000 +/** 文档存储的租约时长（DO 存储忽略） */ +export const TASK_LEASE_MS = 90 * 1000 +/** 连续多少轮 tick 无进展就熔断 */ +const MAX_STALLED_ATTEMPTS = 3 + +export interface TickOptions { + budgetMs: number + concurrency: number + owner: string +} + +export interface TickResult { + ran: number + remaining: number + progressed: boolean +} + +function retryDelayMs(retry: number): number { + return Math.min(5 * 60 * 1000, 5000 * Math.pow(3, Math.max(0, retry - 1))) +} + +function errorText(err: any): string { + const message = String(err?.message || err || "unknown error") + return message.length > 2000 ? `${message.slice(0, 2000)}…` : message +} + +/** + * 执行一个已认领的任务，直到预算用尽或任务结束。 + * 返回「本轮是否有实际进展」，用于熔断与 tick 统计。 + */ +async function executeTask( + store: TaskRunnerStore, + rt: TaskRuntime, + initial: TaskRecord, + deadline: number, + settings: TaskSettings, + owner: string, +): Promise { + const handler = getHandler(initial.type) + if (!handler) { + await store.patch(initial.type, initial.id, { + state: TASK_STATE.failed, + status: "unsupported task type", + error: `no handler registered for task type "${initial.type}"`, + end_time: new Date().toISOString(), + updated_at: Date.now(), + lease: null, + }) + return false + } + + let record = initial + let totalBytes = record.total_bytes + let doneBytes = record.done_bytes + let progressed = false + let lastFlush = 0 + let lastCancelCheck = 0 + + // 每个任务一份 runtime 副本：checkpoint 只能写自己的 cursor + const taskRt: TaskRuntime = { + ...rt, + checkpoint: async (cursor: unknown) => { + record = { ...record, cursor } + await store.patch(initial.type, initial.id, { + cursor, + updated_at: Date.now(), + lease: { owner, expires_at: Date.now() + TASK_LEASE_MS }, + }) + }, + } + + const flush = async (extra: Partial = {}) => { + const progress = + extra.progress ?? computeProgress(doneBytes, totalBytes) + const patch: Partial = { + cursor: record.cursor, + total_bytes: totalBytes, + done_bytes: doneBytes, + progress, + updated_at: Date.now(), + lease: { + owner, + expires_at: Date.now() + TASK_LEASE_MS, + }, + ...extra, + } + record = { ...record, ...patch } + lastFlush = Date.now() + await store.patch(initial.type, initial.id, patch) + } + + try { + while (Date.now() < deadline) { + // 取消检查：每 3s 读一次最新状态（读比写便宜得多） + if (Date.now() - lastCancelCheck > CANCEL_CHECK_MS) { + lastCancelCheck = Date.now() + const fresh = await store.get(initial.type, initial.id) + if (!fresh) return progressed + if (fresh.state === TASK_STATE.canceling) { + record = { ...record, ...fresh } + await store.patch(initial.type, initial.id, { + state: TASK_STATE.canceled, + status: "canceled", + end_time: new Date().toISOString(), + updated_at: Date.now(), + lease: null, + }) + return progressed + } + record = { ...record, ...fresh, cursor: record.cursor } + } + + const result = await handler.runUnit(taskRt, record, deadline) + progressed = true + + if (result.cursor !== undefined) record.cursor = result.cursor + if (typeof result.totalBytes === "number") { + totalBytes = Math.max(totalBytes, result.totalBytes) + } + if (typeof result.doneBytes === "number") { + doneBytes = Math.max(doneBytes, result.doneBytes) + } + + if (result.finished) { + await flush({ + state: TASK_STATE.running, + status: result.status || record.status, + }) + const verdict = await handler.verify(taskRt, record) + const now = Date.now() + if (verdict.ok) { + await store.patch(initial.type, initial.id, { + state: TASK_STATE.succeeded, + status: result.status || record.status || "completed", + progress: 100, + done_bytes: totalBytes > 0 ? totalBytes : doneBytes, + verified: true, + error: "", + end_time: new Date(now).toISOString(), + updated_at: now, + lease: null, + }) + } else { + await store.patch(initial.type, initial.id, { + state: TASK_STATE.failed, + status: "verification failed", + error: verdict.error || "post-transfer verification failed", + verified: false, + end_time: new Date(now).toISOString(), + updated_at: now, + lease: null, + }) + } + return true + } + + if (Date.now() - lastFlush > PROGRESS_FLUSH_MS) { + await flush({ state: TASK_STATE.running, status: result.status ?? record.status }) + } else if (result.status !== undefined) { + record = { ...record, status: result.status } + } + } + + // 预算用尽：保存游标并续租，下一次 tick 从这里继续 + await flush({ state: TASK_STATE.running }) + return progressed + } catch (err: any) { + const now = Date.now() + const retry = (initial.retry || 0) + 1 + const pending = err?.pending + const cursor = pending + ? { ...(record.cursor as any), upload: pending } + : record.cursor + const attempts = (initial.attempts || 0) + 1 + const stalled = !progressed && attempts >= MAX_STALLED_ATTEMPTS + // 记录里的 max_retry 在入队时已由设置填充；这里必须尊重它（包括 0）。 + // 历史实现用 Math.max 会让「用户把重试设为 0」失效。 + const maxRetry = Number.isFinite(initial.max_retry) + ? initial.max_retry + : settings.maxRetry[initial.type] || 0 + + if (!stalled && retry <= maxRetry) { + await store.patch(initial.type, initial.id, { + state: TASK_STATE.waitingRetry, + retry, + attempts, + cursor, + error: errorText(err), + status: `retry ${retry}/${maxRetry} scheduled`, + next_run_at: now + retryDelayMs(retry), + updated_at: now, + lease: null, + }) + } else { + await store.patch(initial.type, initial.id, { + state: TASK_STATE.failed, + retry, + attempts, + cursor, + error: stalled + ? `stalled: no progress after ${attempts} attempts; last error: ${errorText(err)}` + : errorText(err), + status: stalled ? "stalled" : "failed", + end_time: new Date(now).toISOString(), + updated_at: now, + lease: null, + }) + } + return false + } +} + +/** + * 跑一个 tick。 + * + * @param budgetMs 本次 tick 的墙钟预算（会再按 deadline 精确截断） + * @param concurrency 并发任务数（1~4；受「每次调用 6 条出站连接」限制） + */ +export async function runTick( + store: TaskRunnerStore, + rt: TaskRuntime, + options: TickOptions, +): Promise { + const settings = await getTaskSettings() + if (!settings.enabled) { + return { ran: 0, remaining: 0, progressed: false } + } + const deadline = Date.now() + Math.max(0, options.budgetMs) + let ran = 0 + let progressed = false + let stop = false + + const worker = async () => { + while (!stop && Date.now() < deadline) { + let claimed: TaskRecord | null = null + try { + claimed = await store.claim({ + types: RUNNABLE_TYPES, + now: Date.now(), + owner: options.owner, + leaseMs: TASK_LEASE_MS, + staleMs: TASK_STALE_MS, + }) + } catch (err) { + console.error("[Task] claim failed:", err) + stop = true + return + } + if (!claimed) { + stop = true + return + } + ran += 1 + if (isTerminalState(claimed.state)) continue + const did = await executeTask( + store, + rt, + claimed, + deadline, + settings, + options.owner, + ) + if (did) progressed = true + } + } + + const concurrency = Math.min(4, Math.max(1, options.concurrency || 1)) + await Promise.all(Array.from({ length: concurrency }, () => worker())) + + let remaining = 0 + try { + remaining = await store.countActive() + } catch { + remaining = 0 + } + return { ran, remaining, progressed } +} + +/** 是否有可继续执行的任务（DO 用来决定是否继续排 alarm） */ +export async function hasRunnableWork(store: TaskRunnerStore): Promise { + const settings = await getTaskSettings() + if (!settings.enabled) return false + return (await store.countActive()) > 0 +} diff --git a/src/backend/internal/task/settings.ts b/src/backend/internal/task/settings.ts new file mode 100644 index 00000000..34d76e6e --- /dev/null +++ b/src/backend/internal/task/settings.ts @@ -0,0 +1,123 @@ +/** + * 任务系统的设置读取。 + * + * 全部设置项都在 `defaultDb.settings` 中声明（group 11 = TRAFFIC，与 Go 版 + * 的任务设置同组），用户可在设置页直接改。这里只做「字符串 → 数值/布尔」 + * 的解析与默认值兜底，缺项或非法值一律回落默认，避免一个坏值把队列卡死。 + */ +import { getSettings } from "../model/db" +import type { TaskType } from "./types" + +export interface TaskSettings { + enabled: boolean + /** 同步阈值（字节） */ + syncThresholdBytes: number + probeMaxEntries: number + /** 0 = 自动（按泵选择保守值） */ + tickBudgetMs: number + parallelUnits: number + keepDone: number + maxRetry: Record +} + +const DEFAULT_MAX_RETRY: Record = { + upload: 1, + copy: 2, + move: 2, + offline_download: 2, + offline_download_transfer: 2, + decompress: 2, + decompress_upload: 1, + seed_generate: 2, + seed_save: 1, +} + +function asBool(value: unknown, fallback: boolean): boolean { + if (value === undefined || value === null || value === "") return fallback + const text = String(value).trim().toLowerCase() + if (["true", "1", "yes", "on"].includes(text)) return true + if (["false", "0", "no", "off"].includes(text)) return false + return fallback +} + +function asInt(value: unknown, fallback: number, min = 0, max = Number.MAX_SAFE_INTEGER): number { + const parsed = Number(value) + if (!Number.isSafeInteger(parsed)) return fallback + return Math.min(max, Math.max(min, parsed)) +} + +export async function getTaskSettings(): Promise { + let settings: Record = {} + try { + settings = await getSettings() + } catch (err) { + // 配置读取失败时不能抛出：任务 API 仍需返回可读错误而不是 500 空响应 + console.error("[Task] failed to read settings, using defaults:", err) + } + const thresholdMb = asInt(settings.task_sync_threshold_mb, 32, 0, 1024 * 1024) + return { + enabled: asBool(settings.task_enabled, true), + syncThresholdBytes: thresholdMb * 1024 * 1024, + probeMaxEntries: asInt(settings.task_probe_max_entries, 200, 10, 5000), + tickBudgetMs: asInt(settings.task_tick_budget_ms, 0, 0, 60000), + parallelUnits: asInt(settings.task_parallel_units, 1, 1, 4), + keepDone: asInt(settings.task_keep_done, 50, 0, 200), + maxRetry: { + ...DEFAULT_MAX_RETRY, + copy: asInt(settings.task_copy_max_retry, DEFAULT_MAX_RETRY.copy, 0, 10), + move: asInt(settings.task_move_max_retry, DEFAULT_MAX_RETRY.move, 0, 10), + decompress: asInt( + settings.task_decompress_max_retry, + DEFAULT_MAX_RETRY.decompress, + 0, + 10, + ), + offline_download: asInt( + settings.task_download_max_retry, + DEFAULT_MAX_RETRY.offline_download, + 0, + 10, + ), + seed_generate: asInt( + settings.task_seed_max_retry, + DEFAULT_MAX_RETRY.seed_generate, + 0, + 10, + ), + seed_save: asInt( + settings.task_seed_max_retry, + DEFAULT_MAX_RETRY.seed_save, + 0, + 10, + ), + }, + } +} + +/** + * 单次 tick 的时间预算。 + * + * 用户显式配置（200~60000ms）优先；未配置时按「泵」给保守默认： + * 免费计划 CPU 只有 10ms，2s 的 I/O 切片已经接近安全上限；付费计划建议 + * 用户在设置里改成 20000。 + */ +export type TaskPump = "do" | "cron" | "http" | "inline" | "node" | "esa" + +const AUTO_BUDGET: Record = { + do: 2000, + cron: 2000, + http: 5000, + inline: 2000, + node: 30000, + esa: 2000, +} + +export function resolveTickBudget( + settings: TaskSettings, + pump: TaskPump, +): number { + if (settings.tickBudgetMs > 0) { + return Math.min(60000, Math.max(200, settings.tickBudgetMs)) + } + return AUTO_BUDGET[pump] ?? 2000 +} diff --git a/src/backend/internal/task/store.ts b/src/backend/internal/task/store.ts new file mode 100644 index 00000000..b11ef85d --- /dev/null +++ b/src/backend/internal/task/store.ts @@ -0,0 +1,132 @@ +/** + * 任务存储抽象。 + * + * 有两套实现： + * 1. `store_do.ts`——Durable Object 内置 SQLite（Cloudflare 首选，强一致、 + * 行级更新、alarm 自驱）； + * 2. `store_doc.ts`——把每个类型存成一份 JSON 文档（EdgeOne / ESA / Vercel / + * Node 等没有 DO 的运行时）。 + * + * 两者都实现 `TaskRunnerStore`，因此 runner 与各 handler 只依赖接口，不感知 + * 具体后端。API 层用 `TaskAdmin`（不需要 claim）。 + */ +import type { TaskRecord, TaskType } from "./types" + +export interface TaskListQuery { + type: TaskType + states?: number[] + creator?: number + limit?: number +} + +export interface TaskClaimQuery { + /** 按顺序尝试的类型 */ + types: TaskType[] + now: number + owner: string + /** 文档存储的租约时长；DO 实现忽略（用 updated_at 判僵尸） */ + leaseMs: number + /** running 超过该时长视为僵尸任务，可被重新认领 */ + staleMs: number +} + +/** API 侧需要的操作 */ +export interface TaskAdmin { + enqueue(record: TaskRecord): Promise + list(query: TaskListQuery): Promise + /** 按 type 定位文档（DO 实现忽略 type，主键全局唯一） */ + get(type: TaskType, id: string): Promise + /** + * 列级更新：只写传入的字段，避免把 runner 刚写入的进度覆盖掉。 + * 文档存储按 type 定位文档；DO 存储忽略 type。 + */ + patch( + type: TaskType, + id: string, + patch: Partial, + ): Promise + remove(type: TaskType, id: string): Promise + removeMany(entries: Array<{ type: TaskType; id: string }>): Promise + countActive(creatorId?: number): Promise + /** 淘汰该类型的已完成任务，只保留最近 keepDone 条 */ + prune?(type: TaskType, keepDone: number): Promise +} + +/** runner 侧额外需要原子认领 */ +export interface TaskRunnerStore extends TaskAdmin { + claim(query: TaskClaimQuery): Promise +} + +/** 是否到点可执行（不含 running 僵尸判定，那需要 staleMs） */ +export function isRunnableAt(record: TaskRecord, now: number): boolean { + if (record.state === 0 || record.state === 8) { + return (record.next_run_at || 0) <= now + } + return false +} + +/** + * 选择可认领的任务。 + * + * - pending / waiting_retry：到点即可； + * - running：只有「租约过期」或「updated_at 超过 staleMs」才算僵尸。 + * 这条是崩溃恢复的关键：isolate 被杀后没有 finally，只能靠超时判定。 + */ +export function pickRunnable( + tasks: TaskRecord[], + query: TaskClaimQuery, +): TaskRecord | null { + const now = query.now + for (const task of tasks) { + if (task.state === 0 || task.state === 8) { + if ((task.next_run_at || 0) <= now) return task + continue + } + if (task.state === 1) { + const leaseExpired = + !task.lease || !task.lease.expires_at || task.lease.expires_at <= now + const stale = now - (task.updated_at || 0) > query.staleMs + if (leaseExpired || stale) return task + } + } + return null +} + +/** + * 淘汰已完成任务，保留最近 `keepDone` 条。 + * + * 只淘汰终态，绝不淘汰活跃任务——否则用户会看到「任务凭空消失」。 + */ +export function pruneDoneTasks( + tasks: TaskRecord[], + keepDone: number, +): TaskRecord[] { + const done: TaskRecord[] = [] + const keep: TaskRecord[] = [] + for (const task of tasks) { + if (task.state === 2 || task.state === 4 || task.state === 7) { + done.push(task) + } else { + keep.push(task) + } + } + if (done.length <= keepDone) return tasks + done.sort((a, b) => (b.updated_at || 0) - (a.updated_at || 0)) + return [...keep, ...done.slice(0, keepDone)] +} + +/** 单类型文档的任务条数上限：防止一份文档无限增长（KV/D1 都有大小限制） */ +export const MAX_TASKS_PER_DOC = 200 + +export function capTasks(tasks: TaskRecord[]): TaskRecord[] { + if (tasks.length <= MAX_TASKS_PER_DOC) return tasks + // 活跃任务永远不裁（裁掉会让用户以为任务丢失），只用剩余名额保留终态任务 + const active = tasks.filter( + (t) => t.state !== 2 && t.state !== 4 && t.state !== 7, + ) + const done = tasks + .filter((t) => t.state === 2 || t.state === 4 || t.state === 7) + .sort((a, b) => (b.updated_at || 0) - (a.updated_at || 0)) + const room = Math.max(0, MAX_TASKS_PER_DOC - active.length) + return [...active, ...done.slice(0, room)] +} diff --git a/src/backend/internal/task/store_do_client.ts b/src/backend/internal/task/store_do_client.ts new file mode 100644 index 00000000..47d1450b --- /dev/null +++ b/src/backend/internal/task/store_do_client.ts @@ -0,0 +1,80 @@ +/** + * Durable Object 任务存储的客户端（在 Worker 请求侧使用）。 + * + * 真正的数据在 TaskRunner DO 的 SQLite 里；这里只做 RPC 转发。 + * DO 的存在让「入队即 setAlarm」成为可能：入队请求返回后任务立刻开始执行， + * 不必等下一分钟的 cron。 + */ +import type { TaskAdmin, TaskListQuery } from "./store" +import type { TaskRecord, TaskType } from "./types" + +export interface TaskDoNamespace { + idFromName(name: string): any + get(id: any): any +} + +/** 从 env / globalThis 探测 TASK_DO 绑定（EdgeOne 把绑定注入为全局） */ +export function getTaskDoNamespace(env?: any): TaskDoNamespace | null { + const g = typeof globalThis !== "undefined" ? (globalThis as any) : {} + const candidates = [env?.TASK_DO, g?.TASK_DO] + for (const candidate of candidates) { + if (candidate && typeof candidate.idFromName === "function") { + return candidate as TaskDoNamespace + } + } + return null +} + +export function getTaskDoStub(env?: any): any | null { + const namespace = getTaskDoNamespace(env) + if (!namespace) return null + const name = String(env?.TASK_DO_ID || "openlist-tasks") + try { + return namespace.get(namespace.idFromName(name)) + } catch (err) { + console.warn("[Task] failed to get TaskRunner stub:", err) + return null + } +} + +export function hasTaskDo(env?: any): boolean { + return getTaskDoNamespace(env) !== null +} + +export class DoTaskClient implements TaskAdmin { + constructor(private readonly stub: any) {} + + async enqueue(record: TaskRecord): Promise { + await this.stub.enqueue(record) + } + + async list(query: TaskListQuery): Promise { + return (await this.stub.list(query)) || [] + } + + async get(type: TaskType, id: string): Promise { + return (await this.stub.get(type, id)) || null + } + + async patch( + type: TaskType, + id: string, + patch: Partial, + ): Promise { + return (await this.stub.patch(type, id, patch)) || null + } + + async remove(type: TaskType, id: string): Promise { + await this.stub.remove(type, id) + } + + async removeMany( + entries: Array<{ type: TaskType; id: string }>, + ): Promise { + await this.stub.removeMany(entries) + } + + async countActive(creatorId?: number): Promise { + return Number(await this.stub.countActive(creatorId)) || 0 + } +} diff --git a/src/backend/internal/task/store_doc.ts b/src/backend/internal/task/store_doc.ts new file mode 100644 index 00000000..06181e8e --- /dev/null +++ b/src/backend/internal/task/store_doc.ts @@ -0,0 +1,275 @@ +/** + * 文档式任务存储（无 Durable Objects 的运行时）。 + * + * 每个任务类型一份 JSON 文档：key = `tasks_`。 + * - 只用 Driver 的 get/put，因此 KV / Blob / D1 / MySQL / DO 都能工作； + * - key 只用 `[A-Za-z0-9_]`，满足 EdgeOne KV 的最严格约束； + * - **不并入配置对象**，避免与 saveDb() 的整库重写互相覆盖。 + * + * 已知限制：KV/Blob 没有 CAS，并发 isolate 之间可能丢更新。缓解手段是 + * （1）单类型单文档，写入窗口短；（2）claim 使用租约； + * （3）SQL 驱动若提供 casPut 则自动走 CAS（见 writeDoc）。 + */ +import { getKvAccessor, type KvAccessor } from "../model/store/backend" +import { + capTasks, + pruneDoneTasks, + pickRunnable, + type TaskAdmin, + type TaskClaimQuery, + type TaskListQuery, + type TaskRunnerStore, +} from "./store" +import { isDoneState, RUNNABLE_TYPES, type TaskRecord, type TaskType } from "./types" + +interface TaskDoc { + version: number + updated_at: number + tasks: TaskRecord[] +} + +const KEY_PREFIX = "tasks_" + +function docKey(type: TaskType): string { + return `${KEY_PREFIX}${type}` +} + +function emptyDoc(): TaskDoc { + return { version: 0, updated_at: 0, tasks: [] } +} + +function parseDoc(raw: string | null): TaskDoc { + if (!raw) return emptyDoc() + try { + const parsed = JSON.parse(raw) + const tasks = Array.isArray(parsed?.tasks) ? parsed.tasks : [] + return { + version: Number(parsed?.version) || 0, + updated_at: Number(parsed?.updated_at) || 0, + tasks: tasks.filter((t: any) => t && typeof t.id === "string"), + } + } catch (err) { + // 文档损坏不能静默清空后写回（会丢任务），只本次按空处理并告警。 + console.error("[TaskStore] failed to parse task document:", err) + return emptyDoc() + } +} + +export class DocTaskStore implements TaskRunnerStore, TaskAdmin { + private constructor( + private readonly accessor: KvAccessor, + private readonly env: any, + ) {} + + static async create(env?: any, accessor?: KvAccessor): Promise { + const resolved = accessor || (await getKvAccessor(env)) + return new DocTaskStore(resolved, env) + } + + private async read(type: TaskType): Promise<{ doc: TaskDoc; raw: string | null }> { + const raw = await this.accessor.get(docKey(type), this.env) + return { doc: parseDoc(raw), raw } + } + + /** + * 写入文档。若底层提供 compareAndSwap，则用「旧原文」做 CAS； + * 返回 false 表示冲突，调用方应重读后重试。 + */ + private async write( + type: TaskType, + doc: TaskDoc, + expectedRaw: string | null, + ): Promise { + const next: TaskDoc = { + version: (doc.version || 0) + 1, + updated_at: Date.now(), + tasks: capTasks(doc.tasks), + } + const payload = JSON.stringify(next) + const cas = (this.accessor as any).casPut + if (typeof cas === "function") { + return Boolean(await cas.call(this.accessor, docKey(type), expectedRaw, payload, this.env)) + } + await this.accessor.put(docKey(type), payload, this.env) + return true + } + + /** 读改写；CAS 冲突时重试。fn 返回 null 表示「无需变更」 */ + private async mutate( + type: TaskType, + fn: (doc: TaskDoc) => TaskDoc | null, + ): Promise { + for (let attempt = 0; attempt < 3; attempt++) { + const { doc, raw } = await this.read(type) + const next = fn(doc) + if (!next) return false + if (await this.write(type, next, raw)) return true + // 冲突：稍等再重读，避免两个实例在同一毫秒内反复撞车 + await new Promise((resolve) => setTimeout(resolve, 20 * (attempt + 1))) + } + return false + } + + async enqueue(record: TaskRecord): Promise { + await this.mutate(record.type, (doc) => { + const tasks = doc.tasks.filter((t) => t.id !== record.id) + tasks.unshift(record) + return { ...doc, tasks } + }) + } + + async list(query: TaskListQuery): Promise { + const { doc } = await this.read(query.type) + const limit = Math.min(Math.max(1, query.limit ?? 200), 500) + return doc.tasks + .filter((t) => { + if (query.states && !query.states.includes(t.state)) return false + if ( + query.creator !== undefined && + query.creator !== null && + t.creator_id !== query.creator + ) { + return false + } + return true + }) + .sort((a, b) => (b.created_at || 0) - (a.created_at || 0)) + .slice(0, limit) + } + + async get(type: TaskType, id: string): Promise { + const { doc } = await this.read(type) + return doc.tasks.find((t) => t.id === id) || null + } + + async patch( + type: TaskType, + id: string, + patch: Partial, + ): Promise { + let updated: TaskRecord | null = null + await this.mutate(type, (doc) => { + const index = doc.tasks.findIndex((t) => t.id === id) + if (index < 0) return null + const next: TaskRecord = { + ...doc.tasks[index], + ...patch, + updated_at: patch.updated_at ?? Date.now(), + } + updated = next + const tasks = [...doc.tasks] + tasks[index] = next + return { ...doc, tasks } + }) + return updated + } + + async remove(type: TaskType, id: string): Promise { + await this.mutate(type, (doc) => { + if (!doc.tasks.some((t) => t.id === id)) return null + return { ...doc, tasks: doc.tasks.filter((t) => t.id !== id) } + }) + } + + async removeMany(entries: Array<{ type: TaskType; id: string }>): Promise { + const byType = new Map>() + for (const entry of entries) { + const set = byType.get(entry.type) || new Set() + set.add(entry.id) + byType.set(entry.type, set) + } + for (const [type, ids] of byType) { + await this.mutate(type, (doc) => { + const tasks = doc.tasks.filter((t) => !ids.has(t.id)) + if (tasks.length === doc.tasks.length) return null + return { ...doc, tasks } + }) + } + } + + async countActive(creatorId?: number): Promise { + let total = 0 + for (const type of RUNNABLE_TYPES) { + const { doc } = await this.read(type) + total += doc.tasks.filter( + (t) => + !isDoneState(t.state) && + (creatorId === undefined || t.creator_id === creatorId), + ).length + } + return total + } + + /** 按保留数淘汰已完成任务（manager 在入队后调用） */ + async prune(type: TaskType, keepDone: number): Promise { + await this.mutate(type, (doc) => { + const tasks = pruneDoneTasks(doc.tasks, keepDone) + if (tasks.length === doc.tasks.length) return null + return { ...doc, tasks } + }) + } + + async claim(query: TaskClaimQuery): Promise { + for (const type of query.types) { + // 同一个类型内最多尝试 5 次：僵尸任务累计 attempts 超限后会被直接判失败， + // 然后继续找下一个候选，不能让一个卡死的任务把队列堵住。 + for (let attempt = 0; attempt < 5; attempt++) { + const { doc, raw } = await this.read(type) + const candidate = pickRunnable(doc.tasks, query) + if (!candidate) break + + const now = query.now + if (candidate.state === 1) { + const attempts = (candidate.attempts || 0) + 1 + if (attempts > 3) { + const failed: TaskRecord = { + ...candidate, + state: 7, + status: "stalled", + error: `stalled: no completion after ${attempts} attempts`, + end_time: new Date(now).toISOString(), + updated_at: now, + attempts, + lease: null, + } + const next: TaskDoc = { + ...doc, + tasks: doc.tasks.map((t) => (t.id === candidate.id ? failed : t)), + } + if (!(await this.write(type, next, raw))) break + continue + } + const claimed: TaskRecord = { + ...candidate, + state: 1, + start_time: candidate.start_time || new Date(now).toISOString(), + updated_at: now, + attempts, + lease: { owner: query.owner, expires_at: now + query.leaseMs }, + } + const next: TaskDoc = { + ...doc, + tasks: doc.tasks.map((t) => (t.id === candidate.id ? claimed : t)), + } + if (await this.write(type, next, raw)) return claimed + break + } + + const claimed: TaskRecord = { + ...candidate, + state: 1, + start_time: candidate.start_time || new Date(now).toISOString(), + updated_at: now, + lease: { owner: query.owner, expires_at: now + query.leaseMs }, + } + const next: TaskDoc = { + ...doc, + tasks: doc.tasks.map((t) => (t.id === candidate.id ? claimed : t)), + } + if (await this.write(type, next, raw)) return claimed + break + } + } + return null + } +} diff --git a/src/backend/internal/task/task_runner.test.ts b/src/backend/internal/task/task_runner.test.ts new file mode 100644 index 00000000..85e70f4d --- /dev/null +++ b/src/backend/internal/task/task_runner.test.ts @@ -0,0 +1,195 @@ +import assert from "node:assert/strict" +import { test } from "node:test" + +import { runTick, __setTaskHandlerForTest } from "./runner" +import { DocTaskStore } from "./store_doc" +import { TASK_STATE, createTaskRecord } from "./types" +import type { TaskRecord } from "./types" + +function fakeAccessor() { + const map = new Map() + return { + name: "fake", + async get(key: string) { + return map.get(key) ?? null + }, + async put(key: string, value: string) { + map.set(key, value) + }, + async delete(key: string) { + map.delete(key) + }, + async list(prefix: string) { + return [...map.keys()].filter((key) => key.startsWith(prefix)) + }, + } +} + +function makeTask(overrides: Partial = {}): TaskRecord { + return { + ...createTaskRecord({ + type: "copy", + name: "copy test", + creator_id: 1, + creator: "admin", + creator_role: 2, + max_retry: 0, + payload: { op: "copy", src_dir: "/a", dst_dir: "/b", names: ["x"] }, + cursor: { index: 0 }, + }), + ...overrides, + } +} + +async function makeStore() { + return DocTaskStore.create({}, fakeAccessor() as any) +} + +test("runTick executes units until the handler reports finished and verifies it", async () => { + let units = 0 + __setTaskHandlerForTest("copy", { + type: "copy", + async runUnit() { + units += 1 + if (units >= 3) { + return { finished: true, cursor: { units }, totalBytes: 3, doneBytes: 3 } + } + return { cursor: { units }, totalBytes: 3, doneBytes: units, status: `unit ${units}` } + }, + async verify() { + return { ok: true } + }, + }) + + try { + const store = await makeStore() + const task = makeTask() + await store.enqueue(task) + + const result = await runTick(store, { env: {} }, { + budgetMs: 5000, + concurrency: 1, + owner: "test", + }) + + assert.equal(result.ran, 1) + assert.equal(units, 3) + const saved = await store.get("copy", task.id) + assert.equal(saved?.state, TASK_STATE.succeeded) + assert.equal(saved?.progress, 100) + assert.equal(saved?.verified, true) + assert.ok(saved?.end_time, "finished tasks must have end_time") + assert.equal(saved?.lease, null) + } finally { + __setTaskHandlerForTest("copy", null) + } +}) + +test("runTick fails a task when the handler throws and max_retry is exhausted", async () => { + __setTaskHandlerForTest("copy", { + type: "copy", + async runUnit() { + throw new Error("upstream exploded") + }, + async verify() { + return { ok: true } + }, + }) + + try { + const store = await makeStore() + const task = makeTask({ max_retry: 0 }) + await store.enqueue(task) + + await runTick(store, { env: {} }, { + budgetMs: 5000, + concurrency: 1, + owner: "test", + }) + + const saved = await store.get("copy", task.id) + assert.equal(saved?.state, TASK_STATE.failed) + assert.match(saved?.error || "", /upstream exploded/) + assert.ok(saved?.end_time) + } finally { + __setTaskHandlerForTest("copy", null) + } +}) + +test("runTick schedules a retry instead of failing when retries remain", async () => { + __setTaskHandlerForTest("copy", { + type: "copy", + async runUnit() { + throw new Error("temporary failure") + }, + async verify() { + return { ok: true } + }, + }) + + try { + const store = await makeStore() + const task = makeTask({ max_retry: 2 }) + await store.enqueue(task) + + await runTick(store, { env: {} }, { + budgetMs: 5000, + concurrency: 1, + owner: "test", + }) + + const saved = await store.get("copy", task.id) + assert.equal(saved?.state, TASK_STATE.waitingRetry) + assert.equal(saved?.retry, 1) + assert.ok((saved?.next_run_at || 0) > Date.now(), "retry must be delayed") + } finally { + __setTaskHandlerForTest("copy", null) + } +}) + +test("runTick marks a runnable type without handler as failed", async () => { + const store = await makeStore() + const task = makeTask({ type: "decompress", name: "decompress test" }) + await store.enqueue(task) + + await runTick(store, { env: {} }, { + budgetMs: 5000, + concurrency: 1, + owner: "test", + }) + + const saved = await store.get("decompress", task.id) + assert.equal(saved?.state, TASK_STATE.failed) + assert.match(saved?.error || "", /no handler/) +}) + +test("verify failure prevents the task from being marked succeeded", async () => { + __setTaskHandlerForTest("copy", { + type: "copy", + async runUnit() { + return { finished: true, totalBytes: 1, doneBytes: 1 } + }, + async verify() { + return { ok: false, error: "missing target: /b/x" } + }, + }) + + try { + const store = await makeStore() + const task = makeTask() + await store.enqueue(task) + + await runTick(store, { env: {} }, { + budgetMs: 5000, + concurrency: 1, + owner: "test", + }) + + const saved = await store.get("copy", task.id) + assert.equal(saved?.state, TASK_STATE.failed) + assert.equal(saved?.verified, false) + assert.match(saved?.error || "", /missing target/) + } finally { + __setTaskHandlerForTest("copy", null) + } +}) diff --git a/src/backend/internal/task/task_settings.test.ts b/src/backend/internal/task/task_settings.test.ts new file mode 100644 index 00000000..4c6e1302 --- /dev/null +++ b/src/backend/internal/task/task_settings.test.ts @@ -0,0 +1,57 @@ +import assert from "node:assert/strict" +import { test } from "node:test" + +import { resolveTickBudget, type TaskSettings } from "./settings" + +function settings(overrides: Partial = {}): TaskSettings { + return { + enabled: true, + syncThresholdBytes: 32 * 1024 * 1024, + probeMaxEntries: 200, + tickBudgetMs: 0, + parallelUnits: 1, + keepDone: 50, + maxRetry: { + upload: 1, + copy: 2, + move: 2, + offline_download: 2, + offline_download_transfer: 2, + decompress: 2, + decompress_upload: 1, + seed_generate: 2, + seed_save: 1, + }, + ...overrides, + } +} + +test("explicit tick budget wins over the automatic default", () => { + assert.equal( + resolveTickBudget(settings({ tickBudgetMs: 15000 }), "do"), + 15000, + ) + assert.equal( + resolveTickBudget(settings({ tickBudgetMs: 15000 }), "cron"), + 15000, + ) +}) + +test("explicit tick budget is clamped to a safe range", () => { + assert.equal(resolveTickBudget(settings({ tickBudgetMs: 100 }), "do"), 200) + assert.equal( + resolveTickBudget(settings({ tickBudgetMs: 999999 }), "do"), + 60000, + ) +}) + +test("automatic default is conservative on Cloudflare (free-plan safe)", () => { + assert.equal(resolveTickBudget(settings(), "do"), 2000) + assert.equal(resolveTickBudget(settings(), "cron"), 2000) + assert.equal(resolveTickBudget(settings(), "inline"), 2000) +}) + +test("automatic default is larger where the runtime allows it", () => { + assert.equal(resolveTickBudget(settings(), "http"), 5000) + assert.equal(resolveTickBudget(settings(), "node"), 30000) +}) diff --git a/src/backend/internal/task/task_store.test.ts b/src/backend/internal/task/task_store.test.ts new file mode 100644 index 00000000..a1f01393 --- /dev/null +++ b/src/backend/internal/task/task_store.test.ts @@ -0,0 +1,230 @@ +import assert from "node:assert/strict" +import { test } from "node:test" + +import { DocTaskStore } from "./store_doc" +import { + TASK_STATE, + computeProgress, + createTaskRecord, + toTaskInfo, +} from "./types" +import type { TaskRecord } from "./types" + +/** 内存版 KV 访问器：用来验证文档存储的读写/认领/淘汰语义 */ +function fakeAccessor() { + const map = new Map() + return { + name: "fake", + async get(key: string) { + return map.get(key) ?? null + }, + async put(key: string, value: string) { + map.set(key, value) + }, + async delete(key: string) { + map.delete(key) + }, + async list(prefix: string) { + return [...map.keys()].filter((key) => key.startsWith(prefix)) + }, + dump: () => map, + } +} + +function makeRecord(overrides: Partial = {}): TaskRecord { + const record = createTaskRecord({ + type: "copy", + name: "copy /a to /b", + creator_id: 7, + creator: "alice", + creator_role: 1, + max_retry: 1, + payload: { op: "copy", src_dir: "/a", dst_dir: "/b", names: ["x"] }, + cursor: { index: 0 }, + }) + return { ...record, ...overrides } +} + +async function makeStore() { + const accessor = fakeAccessor() + const store = await DocTaskStore.create({}, accessor as any) + return { store, accessor } +} + +test("enqueue/list/get round-trip keeps creator and payload", async () => { + const { store } = await makeStore() + const record = makeRecord() + await store.enqueue(record) + + const list = await store.list({ type: "copy", states: [TASK_STATE.pending] }) + assert.equal(list.length, 1) + assert.equal(list[0].id, record.id) + assert.equal(list[0].creator_id, 7) + assert.deepEqual((list[0].payload as any).names, ["x"]) + + const got = await store.get("copy", record.id) + assert.ok(got) + assert.equal(got?.name, record.name) +}) + +test("list filters by creator and state", async () => { + const { store } = await makeStore() + await store.enqueue(makeRecord({ id: "t1", creator_id: 1 })) + await store.enqueue(makeRecord({ id: "t2", creator_id: 2 })) + await store.enqueue( + makeRecord({ id: "t3", creator_id: 1, state: TASK_STATE.succeeded }), + ) + + const undone = await store.list({ + type: "copy", + creator: 1, + states: [TASK_STATE.pending], + }) + assert.deepEqual( + undone.map((task) => task.id), + ["t1"], + ) + + const all = await store.list({ type: "copy", states: [TASK_STATE.pending] }) + assert.equal(all.length, 2) +}) + +test("patch only updates provided columns", async () => { + const { store } = await makeStore() + const record = makeRecord({ id: "t1" }) + await store.enqueue(record) + + const patched = await store.patch("copy", "t1", { + progress: 42, + done_bytes: 420, + status: "copying", + }) + assert.equal(patched?.progress, 42) + assert.equal(patched?.done_bytes, 420) + // 未提交的字段必须保持原值 + assert.equal(patched?.creator_id, 7) + assert.deepEqual((patched?.payload as any).names, ["x"]) +}) + +test("claim moves a pending task to running and is exclusive while leased", async () => { + const { store } = await makeStore() + await store.enqueue(makeRecord({ id: "t1" })) + + const now = Date.now() + const first = await store.claim({ + types: ["copy"], + now, + owner: "runner-a", + leaseMs: 60_000, + staleMs: 300_000, + }) + assert.ok(first) + assert.equal(first?.id, "t1") + assert.equal(first?.state, TASK_STATE.running) + assert.equal(first?.lease?.owner, "runner-a") + + const second = await store.claim({ + types: ["copy"], + now: now + 1000, + owner: "runner-b", + leaseMs: 60_000, + staleMs: 300_000, + }) + assert.equal(second, null) +}) + +test("claim re-claims a task whose lease expired", async () => { + const { store } = await makeStore() + await store.enqueue(makeRecord({ id: "t1" })) + const now = Date.now() + await store.claim({ + types: ["copy"], + now, + owner: "runner-a", + leaseMs: 1000, + staleMs: 300_000, + }) + const again = await store.claim({ + types: ["copy"], + now: now + 5000, + owner: "runner-b", + leaseMs: 1000, + staleMs: 300_000, + }) + assert.equal(again?.id, "t1") + assert.equal(again?.lease?.owner, "runner-b") +}) + +test("claim gives up on a task that keeps dying and marks it stalled", async () => { + const { store } = await makeStore() + await store.enqueue(makeRecord({ id: "t1", state: TASK_STATE.running })) + const now = Date.now() + // 每次租约过期都会被重新认领并累加 attempts；第 4 次超过阈值直接失败 + for (let attempt = 0; attempt < 4; attempt++) { + await store.claim({ + types: ["copy"], + now: now + attempt * 10_000, + owner: `runner-${attempt}`, + leaseMs: 1000, + staleMs: 5000, + }) + } + const record = await store.get("copy", "t1") + assert.equal(record?.state, TASK_STATE.failed) + assert.equal(record?.status, "stalled") +}) + +test("prune keeps only the most recent finished tasks", async () => { + const { store } = await makeStore() + for (let i = 0; i < 5; i++) { + await store.enqueue( + makeRecord({ + id: `done-${i}`, + state: TASK_STATE.succeeded, + updated_at: 1000 + i, + }), + ) + } + await store.enqueue(makeRecord({ id: "active" })) + + await store.prune("copy", 2) + const done = await store.list({ + type: "copy", + states: [TASK_STATE.succeeded], + }) + assert.equal(done.length, 2) + assert.deepEqual( + done.map((task) => task.id).sort(), + ["done-3", "done-4"], + ) + const active = await store.get("copy", "active") + assert.ok(active, "active tasks must never be pruned") +}) + +test("countActive ignores finished states", async () => { + const { store } = await makeStore() + await store.enqueue(makeRecord({ id: "t1", creator_id: 1 })) + await store.enqueue( + makeRecord({ id: "t2", creator_id: 1, state: TASK_STATE.running }), + ) + await store.enqueue( + makeRecord({ id: "t3", creator_id: 1, state: TASK_STATE.succeeded }), + ) + assert.equal(await store.countActive(1), 2) + assert.equal(await store.countActive(2), 0) +}) + +test("toTaskInfo never exposes payload or cursor", () => { + const info: any = toTaskInfo(makeRecord()) + assert.equal(info.payload, undefined) + assert.equal(info.cursor, undefined) + assert.equal(info.id.length > 0, true) + assert.equal(info.creator_role, 1) +}) + +test("computeProgress clamps and handles unknown totals", () => { + assert.equal(computeProgress(50, 100), 50) + assert.equal(computeProgress(200, 100), 100) + assert.equal(computeProgress(0, 0), 0) + assert.equal(computeProgress(-5, 100), 0) +}) diff --git a/src/backend/internal/task/types.ts b/src/backend/internal/task/types.ts new file mode 100644 index 00000000..1a1a2eae --- /dev/null +++ b/src/backend/internal/task/types.ts @@ -0,0 +1,231 @@ +/** + * 任务模型：字段与状态数值严格对齐 Go 版（tache + server/handles/task.go）。 + * + * 为什么状态用数值：前端任务页直接展示 `state`，Go 的 tache.State 是 + * 0..9 的整数枚举。若改成字符串，前端会出现无法识别的状态（任务永远显示 + * 「未知」），因此这里保持数值不变。 + */ + +/** 与 github.com/OpenListTeam/tache 的 State 一一对应 */ +export const TASK_STATE = { + pending: 0, + running: 1, + succeeded: 2, + canceling: 3, + canceled: 4, + errored: 5, + failing: 6, + failed: 7, + waitingRetry: 8, + beforeRetry: 9, +} as const + +/** 未完成（undone）状态集合，对齐 Go taskRoute 的 /undone 过滤条件 */ +export const ACTIVE_STATES: number[] = [ + TASK_STATE.pending, + TASK_STATE.running, + TASK_STATE.canceling, + TASK_STATE.errored, + TASK_STATE.failing, + TASK_STATE.waitingRetry, + TASK_STATE.beforeRetry, +] + +/** 已完成（done）状态集合 */ +export const DONE_STATES: number[] = [ + TASK_STATE.canceled, + TASK_STATE.failed, + TASK_STATE.succeeded, +] + +/** 可被 runner 认领的状态 */ +export const RUNNABLE_STATES: number[] = [ + TASK_STATE.pending, + TASK_STATE.running, + TASK_STATE.waitingRetry, +] + +/** + * 任务类型白名单：与 Go `SetupTaskRoute` 注册的分组完全一致。 + * + * `upload`/`decompress_upload`/`offline_download_transfer` 在 TS 端暂无生产者， + * 但必须保留，否则前端任务页切换标签会拿到 404。 + */ +export const TASK_TYPES = [ + "upload", + "copy", + "move", + "offline_download", + "offline_download_transfer", + "decompress", + "decompress_upload", + "seed_generate", + "seed_save", +] as const + +export type TaskType = (typeof TASK_TYPES)[number] + +const TASK_TYPE_SET = new Set(TASK_TYPES) + +export function isTaskType(value: unknown): value is TaskType { + return typeof value === "string" && TASK_TYPE_SET.has(value) +} + +/** + * 当前真正会被调度的类型。 + * + * 顺序即执行优先级：下载/解压优先（用户在前台等待结果），其次 move（源端 + * 会被删除，越早完成越安全），再 copy，最后是种子计算。 + */ +export const RUNNABLE_TYPES: TaskType[] = [ + "offline_download", + "decompress", + "move", + "copy", + "seed_generate", + "seed_save", +] + +export interface TaskLease { + owner: string + expires_at: number +} + +/** 持久化的任务记录（payload/cursor 仅供内部使用，绝不通过 API 返回） */ +export interface TaskRecord { + id: string + type: TaskType + name: string + creator_id: number + creator: string + creator_role: number + state: number + status: string + progress: number + total_bytes: number + done_bytes: number + start_time: string | null + end_time: string | null + error: string + retry: number + max_retry: number + created_at: number + updated_at: number + next_run_at: number + /** 连续无进展次数：用于退避与熔断 */ + attempts: number + /** 完整性校验是否通过（仅 succeeded 前有意义） */ + verified: boolean + cursor: unknown + payload: unknown + lease?: TaskLease | null +} + +/** 对外暴露的任务信息（字段与 Go TaskInfo 完全一致） */ +export interface TaskInfo { + id: string + name: string + creator: string + creator_role: number + state: number + status: string + progress: number + start_time: string | null + end_time: string | null + total_bytes: number + error: string +} + +export function isDoneState(state: number): boolean { + return DONE_STATES.includes(state) +} + +export function isActiveState(state: number): boolean { + return ACTIVE_STATES.includes(state) +} + +export function isTerminalState(state: number): boolean { + return isDoneState(state) +} + +/** + * 转成前端契约对象。 + * + * 注意：payload/cursor 一律不返回（可能含源路径、下载 URL 等敏感信息）。 + * progress 为 NaN 时按 100 处理，与 Go getTaskInfo 的行为一致。 + */ +export function toTaskInfo(record: TaskRecord): TaskInfo { + const progress = Number.isFinite(record.progress) ? record.progress : 100 + return { + id: record.id, + name: record.name, + creator: record.creator || "", + creator_role: Number.isFinite(record.creator_role) + ? record.creator_role + : -1, + state: record.state, + status: record.status || "", + progress, + start_time: record.start_time || null, + end_time: record.end_time || null, + total_bytes: Number.isFinite(record.total_bytes) ? record.total_bytes : 0, + error: record.error || "", + } +} + +/** 新建记录：所有字段给全，避免不同存储实现各自补默认值导致漂移 */ +export function createTaskRecord(input: { + type: TaskType + name: string + creator_id?: number + creator?: string + creator_role?: number + max_retry?: number + payload?: unknown + cursor?: unknown + total_bytes?: number +}): TaskRecord { + const now = Date.now() + return { + id: crypto.randomUUID(), + type: input.type, + name: input.name, + creator_id: input.creator_id ?? 0, + creator: input.creator ?? "", + creator_role: input.creator_role ?? -1, + state: TASK_STATE.pending, + status: "", + progress: 0, + total_bytes: input.total_bytes ?? 0, + done_bytes: 0, + start_time: null, + end_time: null, + error: "", + retry: 0, + max_retry: input.max_retry ?? 2, + created_at: now, + updated_at: now, + next_run_at: now, + attempts: 0, + verified: false, + cursor: input.cursor ?? null, + payload: input.payload ?? null, + lease: null, + } +} + +/** + * 计算进度百分比。 + * + * 优先按字节（跨存储复制/下载/解压都能给出 total_bytes），字节未知时退回 + * 百分比 0（前端会显示「进行中」而不是假的 100%）。完成时调用方显式写 100。 + */ +export function computeProgress( + doneBytes: number, + totalBytes: number, +): number { + if (!Number.isFinite(totalBytes) || totalBytes <= 0) return 0 + const value = (Math.max(0, doneBytes) / totalBytes) * 100 + if (!Number.isFinite(value)) return 0 + return Math.min(100, Math.max(0, value)) +} diff --git a/src/backend/server/fs.ts b/src/backend/server/fs.ts index be7c751e..13e37b47 100644 --- a/src/backend/server/fs.ts +++ b/src/backend/server/fs.ts @@ -50,6 +50,18 @@ import { parseZip, extractZipEntry, ZipArchive } from "../internal/archive/zip" import { assertSafeUrl, getTrustedHosts } from "../pkg/http" import { seedRouter, torrentLegacyRouter } from "./seed" import { r2CanSignDirectUpload } from "../drivers/r2/driver" +import { decideCopyMoveMode } from "../internal/task/decide" +import { + enqueueTask, + kickTaskPump, + recordCompletedTask, +} from "../internal/task/manager" +import { toTaskInfo } from "../internal/task/types" +import { + assertDownloadUrlAllowed, + fileNameFromUrl, + offlineDownloadAllowList, +} from "../internal/task/handlers/offline_download" /** * 该路径所属存储是否禁止目录列表（对齐 Go handles.FsList 的 DisableIndex 判断）。 @@ -115,6 +127,83 @@ const getStorageRequestContext = (c: any) => { return { ...(waitUntil ? { waitUntil } : {}), env } } +/** + * 任务运行时上下文：后台执行没有请求对象，但需要 env / origin / waitUntil。 + * origin 会被写进任务 payload —— 存储驱动返回的相对 raw_url 要靠它解析。 + */ +const taskRuntimeOf = (c: any, requestContext?: any) => ({ + env: c.env, + origin: new URL(c.req.url).origin, + waitUntil: requestContext?.waitUntil, +}) + +/** + * 把 copy / move 入队。 + * + * 每个顶层条目一条任务，与 Go 的 FsCopy/FsMove 一致：这样单条失败不会连累 + * 其它条目，前端也能逐条取消/重试。 + */ +async function enqueueFsTasks( + c: any, + user: any, + requestContext: any, + type: "copy" | "move", + actualSrcDir: string, + actualDstDir: string, + names: string[], +): Promise { + const runtime = taskRuntimeOf(c, requestContext) + const preview: any[] = [] + for (const name of names) { + const record = await enqueueTask(c.env, { + type, + name: `${type} ${actualSrcDir.replace(/\/+$/, "")}/${name} to ${actualDstDir}`, + creator_id: Number(user?.id ?? 0), + creator: String(user?.username || ""), + creator_role: Number(user?.role ?? 1), + payload: { + op: type, + src_dir: actualSrcDir, + dst_dir: actualDstDir, + names: [name], + origin: runtime.origin, + }, + }) + preview.push(toTaskInfo(record)) + } + await kickTaskPump(c.env, runtime) + return preview +} + +/** 同步完成的操作也记一条已完成任务（对齐 Go：done 列表可回溯） */ +function recordSyncCompletion( + c: any, + user: any, + requestContext: any, + type: "copy" | "move", + actualSrcDir: string, + actualDstDir: string, + names: string[], +): void { + const runtime = taskRuntimeOf(c, requestContext) + const promise = recordCompletedTask(c.env, { + type, + name: `${type} ${actualSrcDir.replace(/\/+$/, "")}/${names.join(", ")} to ${actualDstDir}`, + creator_id: Number(user?.id ?? 0), + creator: String(user?.username || ""), + creator_role: Number(user?.role ?? 1), + payload: { + op: type, + src_dir: actualSrcDir, + dst_dir: actualDstDir, + names, + origin: runtime.origin, + }, + }) + if (requestContext?.waitUntil) requestContext.waitUntil(promise) + else void promise +} + // ---- 写操作权限校验 ---- // 游客（未登录 / 无凭证 / token 无效）一律 403，普通用户需具备 // WRITE_CONTENT 权限位，管理员放行。修复「任何人可匿名上传/删除文件」 @@ -908,10 +997,51 @@ fsRouter.post("/move", async (c) => { ] if (await deniedByMetaWriteAcl(c, user, aclPaths)) return permissionDenied(c) - await moveItems(actualSrcDir, actualDstDir, cleanNames, requestContext) - return c.json({ code: 200, message: "success", data: null }) + const decision = await decideCopyMoveMode( + taskRuntimeOf(c, requestContext), + { + op: "move", + srcDir: actualSrcDir, + dstDir: actualDstDir, + names: cleanNames, + }, + ) + if (decision.mode === "sync") { + await moveItems(actualSrcDir, actualDstDir, cleanNames, requestContext) + recordSyncCompletion( + c, + user, + requestContext, + "move", + actualSrcDir, + actualDstDir, + cleanNames, + ) + return c.json({ code: 200, message: "success", data: null }) + } + const tasks = await enqueueFsTasks( + c, + user, + requestContext, + "move", + actualSrcDir, + actualDstDir, + cleanNames, + ) + return c.json({ + code: 200, + message: "success", + data: { + message: `Successfully created ${tasks.length} move task(s)`, + tasks, + }, + }) } catch (e: any) { - return c.json({ code: 500, message: safeErrorMessage(e), data: null }) + return c.json({ + code: Number(e?.status) || 500, + message: safeErrorMessage(e), + data: null, + }) } }) @@ -950,10 +1080,51 @@ fsRouter.post("/copy", async (c) => { ] if (await deniedByMetaWriteAcl(c, user, aclPaths)) return permissionDenied(c) - await copyItems(actualSrcDir, actualDstDir, cleanNames, requestContext) - return c.json({ code: 200, message: "success", data: null }) + const decision = await decideCopyMoveMode( + taskRuntimeOf(c, requestContext), + { + op: "copy", + srcDir: actualSrcDir, + dstDir: actualDstDir, + names: cleanNames, + }, + ) + if (decision.mode === "sync") { + await copyItems(actualSrcDir, actualDstDir, cleanNames, requestContext) + recordSyncCompletion( + c, + user, + requestContext, + "copy", + actualSrcDir, + actualDstDir, + cleanNames, + ) + return c.json({ code: 200, message: "success", data: null }) + } + const tasks = await enqueueFsTasks( + c, + user, + requestContext, + "copy", + actualSrcDir, + actualDstDir, + cleanNames, + ) + return c.json({ + code: 200, + message: "success", + data: { + message: `Successfully created ${tasks.length} copy task(s)`, + tasks, + }, + }) } catch (e: any) { - return c.json({ code: 500, message: safeErrorMessage(e), data: null }) + return c.json({ + code: Number(e?.status) || 500, + message: safeErrorMessage(e), + data: null, + }) } }) @@ -1033,21 +1204,62 @@ fsRouter.post("/add_offline_download", async (c) => { if (!user || user.disabled) { return c.json({ code: 401, message: "Unauthorized", data: null }, 401) } + if (!can(user, PermissionBit.OFFLINE_DOWNLOAD)) return permissionDenied(c) const { path: rawPath, urls } = await c.req.json().catch(() => ({})) + const urlList = (Array.isArray(urls) ? urls : []) + .map((url: unknown) => String(url).trim()) + .filter(Boolean) + if (urlList.length === 0) { + return c.json({ code: 400, message: "No URLs provided", data: null }, 400) + } + const requestContext = getStorageRequestContext(c) const reqPath = getActualPath(user, rawPath || "/") - if (!urls || urls.length === 0) { - return c.json({ code: 400, message: "No URLs provided" }) + if (!isWithinUserRoot(user, reqPath)) return permissionDenied(c) + if (await deniedByMetaWriteAcl(c, user, [reqPath])) return permissionDenied(c) + if (offlineDownloadAllowList(c.env).length === 0) { + return c.json( + { + code: 403, + message: + "offline download is disabled: set ALLOW_OFFLINE_DOWNLOAD to a comma-separated host allowlist", + data: null, + }, + 403, + ) } - return c.json( - { - code: 501, - message: - "capability unavailable: this runtime has no durable offline-download adapter; use /fs/seed/offline_download with an approved source and a streaming-capable target driver", - data: { path: reqPath, accepted: 0 }, - }, - 501, - ) + const runtime = taskRuntimeOf(c, requestContext) + const tasks: any[] = [] + try { + for (const url of urlList) { + // 入队即校验：非法地址不应该先占一个任务位再失败 + assertDownloadUrlAllowed(url, c.env) + const name = fileNameFromUrl(url) + const record = await enqueueTask(c.env, { + type: "offline_download", + name: `download ${name} to ${reqPath}`, + creator_id: Number(user.id ?? 0), + creator: String(user.username || ""), + creator_role: Number(user.role ?? 1), + payload: { + url, + dst_dir: reqPath, + name, + origin: runtime.origin, + }, + }) + tasks.push(toTaskInfo(record)) + } + } catch (e: any) { + const status = Number(e?.status) || 400 + return c.json( + { code: status, message: safeErrorMessage(e), data: null }, + status as any, + ) + } + + await kickTaskPump(c.env, runtime) + return c.json({ code: 200, message: "success", data: { tasks } }) }) fsRouter.post("/search", async (c) => { diff --git a/src/backend/server/task.ts b/src/backend/server/task.ts index 6e0c51ad..0aa813bf 100644 --- a/src/backend/server/task.ts +++ b/src/backend/server/task.ts @@ -1,7 +1,25 @@ import { Hono } from "hono" +import type { Context } from "hono" import { getDb, saveDb } from "../internal/model/db" import { getDriver } from "../internal/op/storage" -import { adminAuthMiddleware, matchCronSecret } from "./middlewares" +import { isAdmin } from "../pkg/permission" +import { + adminAuthMiddleware, + getUserFromContext, + matchCronSecret, +} from "./middlewares" +import { + TaskError, + applyBulkAction, + applyTaskAction, + clearDoneTasks, + getTaskDetail, + listTaskInfos, + retryFailedTasks, + runTaskPump, + type TaskActor, +} from "../internal/task/manager" +import { isTaskType, type TaskType } from "../internal/task/types" export const taskRouter = new Hono() @@ -9,7 +27,7 @@ export const taskRouter = new Hono() // 仅管理员可触发（防止未授权用户探测存储配置/触发资源消耗）。 // EdgeOne Schedules 等平台调度器无法附加 Authorization 头， // 因此复用 JWT_SECRET 作为调度密钥，调度请求可携带该密钥 -//（JSON body / query / X-Cron-Secret 头）通过 matchCronSecret 放行。 +// （JSON body / query / X-Cron-Secret 头）通过 matchCronSecret 放行。 taskRouter.all( "/refresh", async (c, next) => { @@ -94,65 +112,272 @@ taskRouter.all( }, ) -// In-memory or stateless placeholder for task management in serverless -const tasks: Record = { - upload: [], - copy: [], - move: [], - offline_download: [], +// ───────────────────────── 任务管理 API ───────────────────────── +// +// 契约与 Go 版 server/handles/task.go 保持一致，前端任务页无需改动： +// GET /task/:type/undone|done +// POST /task/:type/info|cancel|delete|retry?tid= +// POST /task/:type/cancel_some|delete_some|retry_some (body: string[]) +// POST /task/:type/clear_done|clear_succeeded|retry_failed +// POST /task/run (cron secret / admin) + +function errorResponse(c: Context, err: any) { + if (err instanceof TaskError) { + return c.json( + { code: err.status, message: err.message, data: null }, + err.status as any, + ) + } + const message = String(err?.message || err) + console.error("[Task API]", message) + return c.json({ code: 500, message, data: null }, 500) } -// All task-management endpoints are admin-only (placeholder APIs that may -// later carry real file-operation metadata) -taskRouter.use("*", adminAuthMiddleware) +async function resolveActor(c: Context): Promise { + const user = await getUserFromContext(c) + if (!user) return null + // 对齐 Go 的 AuthNotGuest：游客不能访问任务接口 + if (user.username === "guest") return null + return { + id: Number(user.id ?? 0), + username: user.username || "", + role: Number(user.role ?? 1), + isAdmin: isAdmin(user as any), + } +} -taskRouter.get("/:type/:state", (c) => { +function parseTaskType(c: Context): TaskType | null { const type = c.req.param("type") - const state = c.req.param("state") // "undone" | "done" - const list = tasks[type] || [] - const filtered = list.filter((t) => (state === "done" ? t.done : !t.done)) - return c.json({ - code: 200, - message: "success", - data: filtered, + return isTaskType(type) ? type : null +} + +/** seed_save 与 Go 一致：只展示进度，不支持重试 */ +function allowsRetry(type: TaskType): boolean { + return type !== "seed_save" +} + +taskRouter.get("/:type/:state", async (c) => { + const type = parseTaskType(c) + if (!type) { + return c.json({ code: 404, message: "task type not found", data: null }, 404) + } + const state = c.req.param("state") + if (state !== "undone" && state !== "done") { + return c.json( + { code: 400, message: "state must be 'undone' or 'done'", data: null }, + 400, + ) + } + const actor = await resolveActor(c) + if (!actor) { + return c.json({ code: 401, message: "Unauthorized", data: null }, 401) + } + try { + const data = await listTaskInfos(c.env, type, state, actor) + return c.json({ code: 200, message: "success", data }) + } catch (err) { + return errorResponse(c, err) + } +}) + +taskRouter.post("/:type/info", async (c) => { + const type = parseTaskType(c) + if (!type) { + return c.json({ code: 404, message: "task type not found", data: null }, 404) + } + const actor = await resolveActor(c) + if (!actor) { + return c.json({ code: 401, message: "Unauthorized", data: null }, 401) + } + const tid = c.req.query("tid") || "" + if (!tid) { + return c.json({ code: 400, message: "tid is required", data: null }, 400) + } + try { + const data = await getTaskDetail(c.env, type, tid, actor) + return c.json({ code: 200, message: "success", data }) + } catch (err) { + return errorResponse(c, err) + } +}) + +for (const action of ["cancel", "delete", "retry"] as const) { + taskRouter.post(`/:type/${action}`, async (c) => { + const type = parseTaskType(c) + if (!type) { + return c.json( + { code: 404, message: "task type not found", data: null }, + 404, + ) + } + if (action === "retry" && !allowsRetry(type)) { + return c.json( + { code: 404, message: "retry is not supported for this type", data: null }, + 404, + ) + } + const actor = await resolveActor(c) + if (!actor) { + return c.json({ code: 401, message: "Unauthorized", data: null }, 401) + } + const tid = c.req.query("tid") || "" + if (!tid) { + return c.json({ code: 400, message: "tid is required", data: null }, 400) + } + try { + await applyTaskAction(c.env, type, tid, actor, action) + return c.json({ code: 200, message: "success", data: null }) + } catch (err) { + return errorResponse(c, err) + } + }) +} + +for (const action of ["cancel", "delete", "retry"] as const) { + taskRouter.post(`/:type/${action}_some`, async (c) => { + const type = parseTaskType(c) + if (!type) { + return c.json( + { code: 404, message: "task type not found", data: null }, + 404, + ) + } + if (action === "retry" && !allowsRetry(type)) { + return c.json( + { code: 404, message: "retry is not supported for this type", data: null }, + 404, + ) + } + const actor = await resolveActor(c) + if (!actor) { + return c.json({ code: 401, message: "Unauthorized", data: null }, 401) + } + let ids: string[] = [] + try { + const body = await c.req.json() + if (Array.isArray(body)) ids = body.map((id: any) => String(id)) + } catch { + return c.json( + { code: 400, message: "invalid request format", data: null }, + 400, + ) + } + if (ids.length === 0) { + return c.json( + { code: 400, message: "invalid request format", data: null }, + 400, + ) + } + try { + const data = await applyBulkAction(c.env, type, ids, actor, action) + return c.json({ code: 200, message: "success", data }) + } catch (err) { + return errorResponse(c, err) + } }) +} + +taskRouter.post("/:type/clear_done", async (c) => { + const type = parseTaskType(c) + if (!type) { + return c.json({ code: 404, message: "task type not found", data: null }, 404) + } + const actor = await resolveActor(c) + if (!actor) { + return c.json({ code: 401, message: "Unauthorized", data: null }, 401) + } + try { + await clearDoneTasks(c.env, type, actor, false) + return c.json({ code: 200, message: "success", data: null }) + } catch (err) { + return errorResponse(c, err) + } }) -taskRouter.post("/:type/clear_done", (c) => { - const type = c.req.param("type") - if (tasks[type]) { - tasks[type] = tasks[type].filter((t) => !t.done) +taskRouter.post("/:type/clear_succeeded", async (c) => { + const type = parseTaskType(c) + if (!type) { + return c.json({ code: 404, message: "task type not found", data: null }, 404) + } + const actor = await resolveActor(c) + if (!actor) { + return c.json({ code: 401, message: "Unauthorized", data: null }, 401) + } + try { + await clearDoneTasks(c.env, type, actor, true) + return c.json({ code: 200, message: "success", data: null }) + } catch (err) { + return errorResponse(c, err) } - return c.json({ code: 200, message: "success", data: null }) }) -taskRouter.post("/:type/clear_succeeded", (c) => { - const type = c.req.param("type") - if (tasks[type]) { - tasks[type] = tasks[type].filter((t) => t.state !== "succeeded") +taskRouter.post("/:type/retry_failed", async (c) => { + const type = parseTaskType(c) + if (!type) { + return c.json({ code: 404, message: "task type not found", data: null }, 404) + } + if (!allowsRetry(type)) { + return c.json( + { code: 404, message: "retry is not supported for this type", data: null }, + 404, + ) + } + const actor = await resolveActor(c) + if (!actor) { + return c.json({ code: 401, message: "Unauthorized", data: null }, 401) + } + try { + await retryFailedTasks(c.env, type, actor) + return c.json({ code: 200, message: "success", data: null }) + } catch (err) { + return errorResponse(c, err) } - return c.json({ code: 200, message: "success", data: null }) }) -// 以下接口在 TS Worker 中无意义：TS 不维护持久异步任务队列， -// 所有文件操作均同步完成后立即返回。任务 ID / 进度 / 重试 / 取消 -// 等生命周期管理由 Go 后端负责，此处明确返回 501。 -const unsupportedTaskOp = (c: any) => - c.json( - { - code: 501, - message: - "task lifecycle operations (retry/cancel/delete) are not supported in the TS Worker runtime; " + - "file operations execute synchronously and do not produce persistent tasks", - data: null, - }, - 501, - ) - -taskRouter.post("/:type/retry_failed", unsupportedTaskOp) -taskRouter.post("/:type/retry", unsupportedTaskOp) -taskRouter.post("/:type/retry_some", unsupportedTaskOp) -taskRouter.post("/:type/cancel", unsupportedTaskOp) -taskRouter.post("/:type/cancel_some", unsupportedTaskOp) -taskRouter.post("/:type/delete", unsupportedTaskOp) -taskRouter.post("/:type/delete_some", unsupportedTaskOp) +/** + * 任务泵（外部 cron / 手动触发）。 + * + * EdgeOne 的 schedules 最小间隔是一天，无法当分钟级泵使用；部署在非 CF 平台时 + * 需要外部 cron 每分钟 POST 到这里（带 X-Cron-Secret 或管理员凭证）。 + * 只执行已存在的任务，不接受任何路径/URL 参数。 + */ +taskRouter.post("/run", async (c) => { + let isCron = false + try { + isCron = await matchCronSecret(c) + } catch {} + if (!isCron) { + const actor = await resolveActor(c) + if (!actor?.isAdmin) { + return c.json({ code: 401, message: "Unauthorized", data: null }, 401) + } + } + let budgetMs: number | undefined + try { + const body = await c.req.json() + budgetMs = Number(body?.budget_ms) || undefined + } catch { + // 无 body 或非 JSON：用默认预算 + } + try { + const data = await runTaskPump( + c.env, + { + env: c.env, + origin: new URL(c.req.url).origin, + waitUntil: (() => { + try { + return (promise: Promise) => + c.executionCtx.waitUntil(promise) + } catch { + return undefined + } + })(), + }, + { budgetMs, pump: "http" }, + ) + return c.json({ code: 200, message: "success", data }) + } catch (err) { + return errorResponse(c, err) + } +}) diff --git a/src/backend/server/task_api.test.ts b/src/backend/server/task_api.test.ts new file mode 100644 index 00000000..d6a97689 --- /dev/null +++ b/src/backend/server/task_api.test.ts @@ -0,0 +1,222 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { Hono } from "hono" + +import { saveDb } from "../internal/model/db" +import { enqueueTask } from "../internal/task/manager" +import { setupRouter } from "./router" + +const ADMIN_TOKEN = "TASK_ADMIN_TOKEN" +const CRON_SECRET = "TASK_CRON_SECRET_123456" + +const env: any = { JWT_SECRET: CRON_SECRET } + +function makeApp() { + const api = new Hono() + setupRouter(api) + const app = new Hono() + app.route("/api", api) + return app +} + +async function seedDb() { + await saveDb( + { + settings: [{ key: "token", value: ADMIN_TOKEN }], + users: [ + { + id: 1, + username: "admin", + password: "xxx", + role: 2, + permission: 0, + base_path: "/", + disabled: false, + }, + ], + storages: [], + shares: [], + }, + env, + ) +} + +const adminHeaders = { Authorization: `Bearer ${ADMIN_TOKEN}` } + +test("task API requires authentication", async () => { + await seedDb() + const res = await makeApp().request("/api/task/copy/undone", {}, env) + assert.equal(res.status, 401) +}) + +test("unknown task type is a 404 (same as the Go route layout)", async () => { + await seedDb() + const res = await makeApp().request( + "/api/task/not-a-type/undone", + { headers: adminHeaders }, + env, + ) + assert.equal(res.status, 404) +}) + +test("invalid state is rejected", async () => { + await seedDb() + const res = await makeApp().request( + "/api/task/copy/something", + { headers: adminHeaders }, + env, + ) + assert.equal(res.status, 400) +}) + +test("admin can list undone tasks and gets TaskInfo shape", async () => { + await seedDb() + const record = await enqueueTask(env, { + type: "copy", + name: "copy /a to /b", + creator_id: 1, + creator: "admin", + creator_role: 2, + payload: { op: "copy", src_dir: "/a", dst_dir: "/b", names: ["x"] }, + }) + + const res = await makeApp().request( + "/api/task/copy/undone", + { headers: adminHeaders }, + env, + ) + assert.equal(res.status, 200) + const body: any = await res.json() + assert.equal(body.code, 200) + assert.equal(Array.isArray(body.data), true) + const item = body.data.find((t: any) => t.id === record.id) + assert.ok(item, "enqueued task must be visible") + assert.equal(item.name, "copy /a to /b") + assert.equal(item.creator, "admin") + assert.equal(item.creator_role, 2) + assert.equal(item.state, 0) + assert.equal(item.payload, undefined) + assert.equal(item.cursor, undefined) +}) + +test("a task that never existed returns 404 for info/cancel/delete/retry", async () => { + await seedDb() + const app = makeApp() + for (const endpoint of ["info", "cancel", "delete", "retry"]) { + const res = await app.request( + `/api/task/copy/${endpoint}?tid=missing`, + { method: "POST", headers: adminHeaders }, + env, + ) + assert.equal(res.status, 404, `${endpoint} must be 404`) + } +}) + +test("seed_save does not support retry (matches Go canRetry=false)", async () => { + await seedDb() + const res = await makeApp().request( + "/api/task/seed_save/retry?tid=x", + { method: "POST", headers: adminHeaders }, + env, + ) + assert.equal(res.status, 404) +}) + +test("cancel moves a pending task to canceled immediately", async () => { + await seedDb() + const record = await enqueueTask(env, { + type: "move", + name: "move /a to /b", + creator_id: 1, + creator: "admin", + creator_role: 2, + payload: { op: "move", src_dir: "/a", dst_dir: "/b", names: ["x"] }, + }) + const res = await makeApp().request( + `/api/task/move/cancel?tid=${record.id}`, + { method: "POST", headers: adminHeaders }, + env, + ) + assert.equal(res.status, 200) + const info = await makeApp().request( + `/api/task/move/info?tid=${record.id}`, + { method: "POST", headers: adminHeaders }, + env, + ) + const body: any = await info.json() + assert.equal(body.data.state, 4) + assert.equal(body.data.end_time !== null, true) +}) + +test("clear_done removes finished tasks", async () => { + await seedDb() + const record = await enqueueTask(env, { + type: "copy", + name: "copy /c to /d", + creator_id: 1, + creator: "admin", + creator_role: 2, + payload: { op: "copy", src_dir: "/c", dst_dir: "/d", names: ["y"] }, + }) + // 先取消，让它进入 done 状态 + await makeApp().request( + `/api/task/copy/cancel?tid=${record.id}`, + { method: "POST", headers: adminHeaders }, + env, + ) + const res = await makeApp().request( + "/api/task/copy/clear_done", + { method: "POST", headers: adminHeaders }, + env, + ) + assert.equal(res.status, 200) + + const list = await makeApp().request( + "/api/task/copy/done", + { headers: adminHeaders }, + env, + ) + const body: any = await list.json() + assert.equal(body.data.some((t: any) => t.id === record.id), false) +}) + +test("POST /api/task/run requires the cron secret or an admin", async () => { + await seedDb() + const denied = await makeApp().request( + "/api/task/run", + { method: "POST" }, + env, + ) + assert.equal(denied.status, 401) + + const allowed = await makeApp().request( + "/api/task/run", + { + method: "POST", + headers: { "X-Cron-Secret": CRON_SECRET }, + }, + env, + ) + assert.equal(allowed.status, 200) + const body: any = await allowed.json() + assert.equal(body.code, 200) + assert.equal(typeof body.data.ran, "number") + assert.equal(typeof body.data.remaining, "number") +}) + +test("bulk actions report per-tid errors without failing the whole request", async () => { + await seedDb() + const res = await makeApp().request( + "/api/task/copy/delete_some", + { + method: "POST", + headers: { ...adminHeaders, "Content-Type": "application/json" }, + body: JSON.stringify(["missing-1", "missing-2"]), + }, + env, + ) + assert.equal(res.status, 200) + const body: any = await res.json() + assert.equal(body.data["missing-1"], "task not found") + assert.equal(body.data["missing-2"], "task not found") +}) diff --git a/src/backend/worker.ts b/src/backend/worker.ts index 89b1d898..0359cf35 100644 --- a/src/backend/worker.ts +++ b/src/backend/worker.ts @@ -1,10 +1,39 @@ import app from "./index" import { OpenListDB } from "./durable-objects/OpenListDB" +import { TaskRunner } from "./durable-objects/TaskRunner" +import { setEnvCtx } from "./internal/model/db" +import { runTaskPump } from "./internal/task/manager" -// Durable Object 类（DB_DRIVER=do 时使用），需在 wrangler.toml 声明 -// new_sqlite_classes = ["OpenListDB"] 与对应的 binding。 -export { OpenListDB } +// Durable Object 类（需在 wrangler 配置里声明 binding 与 migrations）： +// OpenListDB —— DB_DRIVER=do 时的配置存储； +// TaskRunner —— 任务队列（SQLite + alarm 自驱）。 +export { OpenListDB, TaskRunner } export default { fetch: app.fetch, + + /** + * Cron Trigger（每分钟）。 + * + * 这里**不做实际工作**，只转发给 TaskRunner DO：cron 只有 10ms CPU， + * 真正的时间片在 DO 里执行（30s CPU / 15min 墙钟）。 + * 有 DO 时它是 alarm 链的看门狗；没有 DO（非 CF 平台）时退化为文档存储的泵。 + */ + async scheduled(event: any, env: any, ctx: any) { + setEnvCtx(env) + const origin = + env?.PUBLIC_ORIGIN || + (typeof process !== "undefined" ? process.env?.PUBLIC_ORIGIN : undefined) + const pump = runTaskPump( + env, + { + env, + origin: origin ? String(origin) : undefined, + waitUntil: (promise: Promise) => ctx.waitUntil(promise), + }, + { pump: "cron" }, + ) + // 显式 catch：cron 的失败不应该以未处理的 rejection 形式丢弃 + ctx.waitUntil(pump.catch((err: any) => console.error("[Task] cron tick failed:", err))) + }, } diff --git a/wrangler.jsonc b/wrangler.jsonc index ec22634f..2fffc660 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -139,4 +139,28 @@ // "r2_buckets": [ // { "binding": "R2", "bucket_name": "your-bucket" } // ], + + // =========================================================================== + // 任务系统（必选，免费计划同样支持） + // + // TaskRunner 用内置 SQLite 存任务、用 alarm 自驱执行：入队后立刻开跑， + // 跑完再排下一次，不再受「请求返回即终止」限制。 + // 没有这个 binding 时自动降级为「任务文档 + cron/请求内时间片」，功能可用 + // 但延迟更高（EdgeOne/ESA/Vercel 等平台走这条路径）。 + // + // 注意：`limits.cpu_ms` 只在付费计划生效；免费计划 CPU 上限很低， + // 因此 `task_tick_budget_ms` 默认 2000ms（可在设置里调整）。 + // =========================================================================== + "durable_objects": { + "bindings": [{ "name": "TASK_DO", "class_name": "TaskRunner" }] + }, + "migrations": [ + { "tag": "v1", "new_sqlite_classes": ["OpenListDB"] }, + { "tag": "v2", "new_sqlite_classes": ["TaskRunner"] } + ], + + // 每分钟一次：仅作为 alarm 链的看门狗（cron 只转发给 DO，几乎不耗 CPU）。 + "triggers": { + "crons": ["* * * * *"] + }, }