diff --git a/.github/workflows/i18n-lint.yml b/.github/workflows/i18n-lint.yml index b2c3449ec..fa3191e7f 100644 --- a/.github/workflows/i18n-lint.yml +++ b/.github/workflows/i18n-lint.yml @@ -6,6 +6,7 @@ on: - 'i18n/**' - 'content/**' - 'hack/check-i18n.sh' + - 'hack/i18n/**' - '.github/workflows/i18n-lint.yml' push: branches: @@ -14,6 +15,7 @@ on: - 'i18n/**' - 'content/**' - 'hack/check-i18n.sh' + - 'hack/i18n/**' - '.github/workflows/i18n-lint.yml' workflow_dispatch: @@ -32,5 +34,23 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Install pyyaml + # check-i18n.sh reads `l10n` through lib.l10n_mode, and the unit tests + # need it too. The Claude SDK is imported lazily at runtime, not here. + run: python3 -m pip install --quiet pyyaml - name: Check i18n key parity and translation freshness run: ./hack/check-i18n.sh + - name: Run i18n pipeline unit tests + # Guards the pure functions of the translation pipeline — including the + # ref/relref deref that keeps a partially-translated page from breaking + # the Hugo build. + run: python3 hack/i18n/test_i18n.py + - name: Typography lint on published translations + # Advisory: reports per-language typography drift (hand edits, pages + # translated before a rule existed) without failing the build. Flip to + # --strict per language once its backlog is clean. + run: python3 hack/i18n/lint_translations.py diff --git a/.github/workflows/i18n-translate.yml b/.github/workflows/i18n-translate.yml new file mode 100644 index 000000000..e9e1b895d --- /dev/null +++ b/.github/workflows/i18n-translate.yml @@ -0,0 +1,111 @@ +name: i18n translate + +# Daily translation run, one weekly PR — the same contract as hack/i18n/run-daily.sh, +# moved off a maintainer's clone and personal Max subscription onto CI and an +# ORG-OWNED Anthropic API key (auth: api-key). Nothing reaches production +# automatically: the run pushes to a weekly branch and opens a PR a maintainer +# merges. No auto-merge, CODEOWNERS/branch protection untouched. +# +# Required secret: +# I18N_ANTHROPIC_API_KEY — an organization-owned Anthropic API key (metered). +# Optional secret: +# I18N_GH_TOKEN — a PAT/app token used to push the branch and open the +# PR so its checks (i18n-lint, DCO) actually run; the +# default GITHUB_TOKEN cannot trigger them. Without it +# the run still works, but the PR opens with no checks. +# +# Until I18N_ANTHROPIC_API_KEY exists the job is a green no-op (see the guard step), +# so enabling the schedule before provisioning the key does not spam failures. +# +# Spend is bounded twice per run: a page limit (DEFAULT_PAGE_LIMIT on the schedule) +# and a wall-clock budget well inside timeout-minutes, so translate.py stops taking +# new pages while there is still time to commit and push what it has. A run killed +# by the timeout would lose every page it translated. Set a spend limit on the +# key's Anthropic workspace as the outer guard. + +on: + schedule: + - cron: '17 3 * * *' # daily, 03:17 UTC + workflow_dispatch: + inputs: + lang: + description: 'Limit to one language code (e.g. ru); empty = all' + required: false + default: '' + limit: + description: 'Max pages this run; empty = the scheduled default (40)' + required: false + default: '' + +concurrency: + group: i18n-translate + cancel-in-progress: false + +permissions: + contents: write + pull-requests: write + +jobs: + translate: + runs-on: ubuntu-latest + timeout-minutes: 300 + env: + I18N_AUTH: api-key + DEFAULT_PAGE_LIMIT: '40' + # Leaves an hour of the 300-minute timeout for the page in flight and the + # commit/push/PR steps. + I18N_TIME_BUDGET_MIN: '240' + CLAUDE_CODE_VERSION: '2.1.287' + steps: + - name: Guard — org API key provisioned + id: guard + env: + ANTHROPIC_API_KEY: ${{ secrets.I18N_ANTHROPIC_API_KEY }} + run: | + if [ -z "${ANTHROPIC_API_KEY}" ]; then + echo "::notice::secret I18N_ANTHROPIC_API_KEY is not set yet — skipping this run." + echo "skip=true" >> "$GITHUB_OUTPUT" + fi + + - name: Checkout + if: steps.guard.outputs.skip != 'true' + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.I18N_GH_TOKEN || github.token }} + + - name: Fetch all branches (weekly-branch bookkeeping needs origin/i18n/week-*) + if: steps.guard.outputs.skip != 'true' + run: | + git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*' + git fetch --prune origin + + - name: Set up Python + if: steps.guard.outputs.skip != 'true' + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Set up Node + if: steps.guard.outputs.skip != 'true' + uses: actions/setup-node@v4 + with: + node-version: '22' + + - name: Install Claude CLI (driven by the Agent SDK) + if: steps.guard.outputs.skip != 'true' + run: npm install -g "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" + + - name: Run translation pipeline + if: steps.guard.outputs.skip != 'true' + env: + # Secrets only reach the step that needs them, not the npm/pip installs. + ANTHROPIC_API_KEY: ${{ secrets.I18N_ANTHROPIC_API_KEY }} + GH_TOKEN: ${{ secrets.I18N_GH_TOKEN || github.token }} + # Dispatch inputs go through env, never interpolated into the script. + INPUT_LANG: ${{ github.event.inputs.lang }} + INPUT_LIMIT: ${{ github.event.inputs.limit }} + run: | + args=(--limit "${INPUT_LIMIT:-$DEFAULT_PAGE_LIMIT}") + if [ -n "${INPUT_LANG}" ]; then args+=(--lang "${INPUT_LANG}"); fi + hack/i18n/run-daily.sh "${args[@]}" diff --git a/.gitignore b/.gitignore index 09ffbbfb3..a36afeff9 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,13 @@ static/docs/*/cozystack-api/ # Claude Code local settings .claude/ + +# i18n pipeline venv (bootstrapped by hack/i18n/run-daily.sh) +.venv-i18n/ + +# i18n pipeline: per-run artifacts, not content +hack/i18n/last-run-findings.md + .demo-build-uidir # demo overlay ships a public/ with the MSW worker — keep it despite the public/ ignore diff --git a/config/production/hugo.yaml b/config/production/hugo.yaml index 03461200f..0f0f83aa1 100644 --- a/config/production/hugo.yaml +++ b/config/production/hugo.yaml @@ -72,6 +72,18 @@ module: files: - '! **/_include/*' - '! docs/next/**' + - source: content/es + target: content + lang: es + files: + - '! **/_include/*' + - '! docs/next/**' + - source: content/pt-br + target: content + lang: pt-br + files: + - '! **/_include/*' + - '! docs/next/**' - source: node_modules/bootstrap target: assets/vendor/bootstrap - source: node_modules/@fortawesome/fontawesome-free diff --git a/content/de/docs/v1.4/getting-started/_index.md b/content/de/docs/v1.4/getting-started/_index.md deleted file mode 100644 index 579ec0477..000000000 --- a/content/de/docs/v1.4/getting-started/_index.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -title: "Erste Schritte mit Cozystack: Eine Private Cloud von Grund auf bereitstellen" -linkTitle: "Erste Schritte" -description: "Machen Sie Ihre ersten Schritte, betreiben Sie ein Homelab und bauen Sie einen PoC mit Cozystack." -weight: 10 -aliases: - - /docs/v1.4/get-started -source_digest: "sha256:0e11921728e455b5e48db02f13a3dacf4a0024b7f556cbe05bfb342384b90a4f" -translation_status: current -l10n: mt ---- - -Dieses Tutorial führt Sie durch Ihre erste Bereitstellung eines Cozystack-Clusters. -Unterwegs lernen Sie zentrale Konzepte kennen, nutzen Cozystack über das Dashboard und die CLI -und erhalten einen funktionierenden Proof of Concept. - -Das Tutorial ist in mehrere Schritte unterteilt. -Schließen Sie jeden Schritt ab, bevor Sie mit dem nächsten beginnen: - -| Schritt | Beschreibung | -|-----------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------| -| [Voraussetzungen: Infrastruktur und Tools vorbereiten](/docs/v1.4/getting-started/requirements/) | Bereiten Sie die Infrastruktur vor und installieren Sie die erforderlichen CLI-Tools auf Ihrem Rechner, bevor Sie dieses Tutorial starten. | -| 1. [Talos Linux installieren](/docs/v1.4/getting-started/install-talos/) | Installieren Sie eine Cozystack-spezifische Distribution von Talos Linux mit [`boot-to-talos`][btt] – wohl die einfachste Installationsmethode. | -| 2. [Kubernetes-Cluster installieren und initialisieren](/docs/v1.4/getting-started/install-kubernetes/) | Initialisieren Sie einen Kubernetes-Cluster mit [Talm][talm], dem für Cozystack entwickelten Tool für das Talos-Konfigurationsmanagement. | -| 3. [Cozystack installieren und konfigurieren](/docs/v1.4/getting-started/install-cozystack/) | Installieren Sie Cozystack, erhalten Sie Administratorzugriff, nehmen Sie die Grundkonfiguration vor und rufen Sie das Cozystack-Dashboard auf. | -| 4. [Tenant für Benutzer und Teams erstellen](/docs/v1.4/getting-started/create-tenant/) | Erstellen Sie einen Benutzer-Tenant – die Grundlage von RBAC in Cozystack – und greifen Sie über das Dashboard und die Cozystack API darauf zu. | -| 5. [Verwaltete Anwendungen bereitstellen](/docs/v1.4/getting-started/deploy-app/) | Beginnen Sie mit der Nutzung von Cozystack: Stellen Sie eine virtuelle Maschine, eine verwaltete Anwendung und einen Kubernetes-Cluster für einen Tenant bereit. | - -[btt]: https://github.com/cozystack/boot-to-talos -[talm]: https://github.com/cozystack/talm diff --git a/content/hi/docs/v1.4/getting-started/_index.md b/content/hi/docs/v1.4/getting-started/_index.md deleted file mode 100644 index 44472ceda..000000000 --- a/content/hi/docs/v1.4/getting-started/_index.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -title: "Cozystack के साथ शुरुआत: शुरू से प्राइवेट क्लाउड परिनियोजित करना" -linkTitle: "शुरुआत करें" -description: "अपने पहले कदम उठाएँ, होम लैब चलाएँ और Cozystack के साथ PoC बनाएँ।" -weight: 10 -aliases: - - /docs/v1.4/get-started -source_digest: "sha256:0e11921728e455b5e48db02f13a3dacf4a0024b7f556cbe05bfb342384b90a4f" -translation_status: current -l10n: mt ---- - -यह ट्यूटोरियल आपको Cozystack क्लस्टर के पहले परिनियोजन में मार्गदर्शन करेगा। -इस दौरान आप प्रमुख अवधारणाओं से परिचित होंगे, डैशबोर्ड और CLI के माध्यम से Cozystack का उपयोग करना सीखेंगे, -और एक कार्यशील PoC प्राप्त करेंगे। - -ट्यूटोरियल को कई चरणों में बाँटा गया है। -अगला चरण शुरू करने से पहले प्रत्येक चरण पूरा करना सुनिश्चित करें: - -| चरण | विवरण | -|-----------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------| -| [आवश्यकताएँ: इन्फ्रास्ट्रक्चर और टूल तैयार करें](/docs/v1.4/getting-started/requirements/) | इस ट्यूटोरियल को शुरू करने से पहले इन्फ्रास्ट्रक्चर तैयार करें और अपनी मशीन पर आवश्यक CLI टूल इंस्टॉल करें। | -| 1. [Talos Linux इंस्टॉल करें](/docs/v1.4/getting-started/install-talos/) | [`boot-to-talos`][btt] का उपयोग करके Cozystack-विशिष्ट Talos Linux वितरण इंस्टॉल करें; यह संभवतः इंस्टॉल करने का सबसे आसान तरीका है। | -| 2. [Kubernetes क्लस्टर इंस्टॉल और बूटस्ट्रैप करें](/docs/v1.4/getting-started/install-kubernetes/) | Cozystack के लिए बनाए गए Talos कॉन्फ़िगरेशन प्रबंधन टूल [Talm][talm] का उपयोग करके Kubernetes क्लस्टर बूटस्ट्रैप करें। | -| 3. [Cozystack इंस्टॉल और कॉन्फ़िगर करें](/docs/v1.4/getting-started/install-cozystack/) | Cozystack इंस्टॉल करें, प्रशासनिक पहुँच प्राप्त करें, बुनियादी कॉन्फ़िगरेशन करें और Cozystack डैशबोर्ड तक पहुँचें। | -| 4. [उपयोगकर्ताओं और टीमों के लिए टेनेंट बनाएँ](/docs/v1.4/getting-started/create-tenant/) | एक उपयोगकर्ता टेनेंट बनाएँ — Cozystack में RBAC की नींव — और डैशबोर्ड तथा Cozystack API के माध्यम से उस तक पहुँचें। | -| 5. [प्रबंधित एप्लिकेशन परिनियोजित करें](/docs/v1.4/getting-started/deploy-app/) | Cozystack का उपयोग शुरू करें: एक वर्चुअल मशीन, प्रबंधित एप्लिकेशन और टेनेंट के लिए Kubernetes क्लस्टर परिनियोजित करें। | - -[btt]: https://github.com/cozystack/boot-to-talos -[talm]: https://github.com/cozystack/talm diff --git a/content/ru/docs/v1.4/getting-started/_index.md b/content/ru/docs/v1.4/getting-started/_index.md deleted file mode 100644 index da37da60f..000000000 --- a/content/ru/docs/v1.4/getting-started/_index.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -title: "Начало работы с Cozystack: развёртывание частного облака с нуля" -linkTitle: "Начало работы" -description: "Сделайте первые шаги, разверните домашнюю лабораторию, соберите PoC на Cozystack." -weight: 10 -aliases: - - /docs/v1.4/get-started -source_digest: "sha256:0e11921728e455b5e48db02f13a3dacf4a0024b7f556cbe05bfb342384b90a4f" -translation_status: current -l10n: mt ---- - -Это руководство проведёт вас через первое развёртывание кластера Cozystack. -По ходу работы вы познакомитесь с ключевыми концепциями, научитесь работать с Cozystack через панель управления и CLI -и получите рабочий PoC. - -Руководство разбито на несколько шагов. -Обязательно завершайте каждый шаг перед началом следующего: - -| Шаг | Описание | -|-----------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------| -| [Требования: подготовка инфраструктуры и инструментов](/docs/v1.4/getting-started/requirements/) | Подготовьте инфраструктуру и установите необходимые CLI-инструменты на локальную машину перед выполнением этого руководства. | -| 1. [Установка Talos Linux](/docs/v1.4/getting-started/install-talos/) | Установите дистрибутив Talos Linux, адаптированный для Cozystack, с помощью [`boot-to-talos`][btt]; это, вероятно, самый простой способ установки. | -| 2. [Установка и инициализация кластера Kubernetes](/docs/v1.4/getting-started/install-kubernetes/) | Инициализируйте кластер Kubernetes с помощью [Talm][talm], инструмента управления конфигурацией Talos, созданного для Cozystack. | -| 3. [Установка и настройка Cozystack](/docs/v1.4/getting-started/install-cozystack/) | Установите Cozystack, получите административный доступ, выполните базовую настройку и откройте панель управления Cozystack. | -| 4. [Создание тенанта для пользователей и команд](/docs/v1.4/getting-started/create-tenant/) | Создайте пользовательский тенант — основу RBAC в Cozystack — и получите доступ к нему через панель управления и Cozystack API. | -| 5. [Развёртывание управляемых приложений](/docs/v1.4/getting-started/deploy-app/) | Начните пользоваться Cozystack: разверните виртуальную машину, управляемое приложение и кластер Kubernetes для тенанта. | - -[btt]: https://github.com/cozystack/boot-to-talos -[talm]: https://github.com/cozystack/talm diff --git a/content/zh-cn/docs/v1.4/getting-started/_index.md b/content/zh-cn/docs/v1.4/getting-started/_index.md deleted file mode 100644 index 3d6f9a926..000000000 --- a/content/zh-cn/docs/v1.4/getting-started/_index.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -title: "Cozystack 入门：从零部署私有云" -linkTitle: "入门" -description: "迈出第一步，搭建家庭实验室，用 Cozystack 构建 PoC。" -weight: 10 -aliases: - - /docs/v1.4/get-started -source_digest: "sha256:0e11921728e455b5e48db02f13a3dacf4a0024b7f556cbe05bfb342384b90a4f" -translation_status: current -l10n: mt ---- - -本教程将引导你完成首次部署 Cozystack 集群。 -在此过程中，你将了解关键概念，学会通过仪表盘和 CLI 使用 Cozystack， -并获得一个可运行的概念验证（PoC）。 - -本教程分为多个步骤。 -请确保完成每一步后再开始下一步： - -| 步骤 | 说明 | -|-----------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------| -| [前置条件：准备基础设施和工具](/docs/v1.4/getting-started/requirements/) | 在运行本教程之前，请准备基础设施并在本机安装所需的 CLI 工具。 | -| 1. [安装 Talos Linux](/docs/v1.4/getting-started/install-talos/) | 使用 [`boot-to-talos`][btt] 安装 Cozystack 专用的 Talos Linux 发行版，这可能是最简单的安装方法。 | -| 2. [安装并引导 Kubernetes 集群](/docs/v1.4/getting-started/install-kubernetes/) | 使用 [Talm][talm]（专为 Cozystack 打造的 Talos 配置管理工具）引导 Kubernetes 集群。 | -| 3. [安装并配置 Cozystack](/docs/v1.4/getting-started/install-cozystack/) | 安装 Cozystack，获取管理员访问权限，完成基本配置，并访问 Cozystack 仪表盘。 | -| 4. [为用户和团队创建租户](/docs/v1.4/getting-started/create-tenant/) | 创建用户租户（Cozystack 中 RBAC 的基础），并通过仪表盘和 Cozystack API 访问该租户。 | -| 5. [部署托管应用](/docs/v1.4/getting-started/deploy-app/) | 开始使用 Cozystack：部署虚拟机、托管应用和租户的 Kubernetes 集群。 | - -[btt]: https://github.com/cozystack/boot-to-talos -[talm]: https://github.com/cozystack/talm diff --git a/hack/check-i18n.sh b/hack/check-i18n.sh index 6299046d3..20a20bdfe 100755 --- a/hack/check-i18n.sh +++ b/hack/check-i18n.sh @@ -4,9 +4,17 @@ # # Subcommands: # check (default) Run every guard; exit non-zero on any gap. -# update-digests Recompute and rewrite `source_digest` in every translated -# page from its English source. Run this after editing an -# English source or after adding/refreshing a translation. +# Hand-localized pages (l10n: transcreate) get a ::warning:: +# instead of an error when they drift: the pipeline never +# regenerates them, so drift is a report for a human, not a +# build failure. +# update-digests [file...] +# Recompute and rewrite `source_digest` in translated pages +# from their English source. Run after editing an English +# source or after adding/refreshing a translation. Without +# arguments, hand-localized pages are SKIPPED (a wholesale +# re-stamp would silence their drift report); pass the file +# explicitly to re-stamp one you refreshed by hand. # # Guards run by `check`: # 1. i18n key parity — every i18n/.toml must define exactly the same @@ -67,6 +75,26 @@ translated_digest_files() { | sort } +# A page a human localized by hand (l10n: transcreate). The pipeline never +# regenerates these, so a stale digest here is a drift REPORT, not a build +# failure — and re-stamping it wholesale would silence the report while the +# drift persists. +# +# The value is read by lib.l10n_mode, the same YAML parser the pipeline uses, so +# the two can never disagree: `l10n: 'transcreate'` and `l10n: transcreate # note` +# are the same value, and a shell regex accepting only some forms would let CI +# treat as machine output a page the pipeline protects. Needs python3 with pyyaml; +# without it the lint stops rather than guessing. +L10N_READER='import sys; sys.path.insert(0, sys.argv[1]); import lib; print(lib.l10n_mode(sys.argv[2]) or "")' +is_transcreate() { + local mode + if ! mode="$(python3 -c "$L10N_READER" "$(dirname "$0")/i18n" "$1" 2>&1)"; then + echo "::error::cannot read l10n from $1 (python3 with pyyaml required): $mode" >&2 + exit 2 + fi + [ "$mode" = "transcreate" ] +} + # Map content// -> content/en/ en_source_for() { local f="$1" @@ -75,6 +103,37 @@ en_source_for() { echo "$CONTENT_DIR/$DEFAULT_LANG/$rel" } +# The latest docs version (params.latest_version_id in hugo.yaml). The pipeline +# only ever refreshes the latest version, so a stale digest on an older, +# noindex'd version is not something a PR can fix by rerunning. Read once and +# cached. lib.py fails closed on a missing key; here we degrade to an empty +# value, which turns is_superseded_docs into a no-op (nothing is treated as +# superseded) rather than silencing a genuine error. +LATEST_DOCS="" +latest_docs_version() { + local hugo; hugo="$(dirname "$CONTENT_DIR")/hugo.yaml" + [ -f "$hugo" ] || return 0 + grep -m1 -E '^[[:space:]]*latest_version_id:' "$hugo" \ + | sed -E "s/.*latest_version_id:[[:space:]]*[\"']?([^\"'[:space:]]+)[\"']?.*/\\1/" +} + +# True when f is a translated docs page of a NON-latest version +# (content//docs//... with ver != latest). Such a page is out of the +# pipeline's scope: it is never refreshed and is removed by the next pipeline +# run (find_orphan_translations), so a drifted digest on it must be advisory, +# not a CI blocker for every unrelated PR in the repo. +is_superseded_docs() { + [ -n "$LATEST_DOCS" ] || return 1 + local rel="${1#"$CONTENT_DIR"/}"; rel="${rel#*/}" # under the lang + case "$rel" in + docs/*/*) + local ver="${rel#docs/}"; ver="${ver%%/*}" + [ "$ver" != "$LATEST_DOCS" ] + ;; + *) return 1 ;; + esac +} + check_key_parity() { local ref="$I18N_DIR/$DEFAULT_LANG.toml" local rc=0 @@ -115,7 +174,12 @@ check_key_parity() { check_digest_freshness() { local rc=0 local checked=0 + local warned=0 local f + LATEST_DOCS="$(latest_docs_version)" + # A value that names no docs tree (a bad parse, a typo) must not mark every + # versioned page as superseded and downgrade real drift to a warning. + [ -d "$CONTENT_DIR/$DEFAULT_LANG/docs/$LATEST_DOCS" ] || LATEST_DOCS="" while IFS= read -r f; do [ -z "$f" ] && continue local src expected actual @@ -129,28 +193,68 @@ check_digest_freshness() { actual="$(grep -m1 -E '^source_digest:' "$f" | sed -E 's/.*sha256:([0-9a-fA-F]+).*/\1/')" checked=$((checked + 1)) if [ "$actual" != "$expected" ]; then - rc=1 - echo "::error::stale translation: $f" - echo " English source: $src" - echo " recorded digest: sha256:${actual}" - echo " current digest: sha256:${expected}" - echo " fix: refresh the translation, then run 'hack/check-i18n.sh update-digests'" + if is_transcreate "$f"; then + # Advisory by design: a drifted transcreation is refreshed when a + # human decides to, and must not block unrelated PRs meanwhile. + warned=$((warned + 1)) + echo "::warning::hand-localized page drifted from its English source: $f" + echo " refresh the transcreation by hand, then re-stamp it with:" + echo " hack/check-i18n.sh update-digests $f" + elif is_superseded_docs "$f"; then + # Out of the pipeline's scope: the latest version is $LATEST_DOCS, the + # pipeline never refreshes older versions, and its next run removes this + # page as a superseded orphan. Failing the lint here would block every + # unrelated PR on a page no PR can fix by rerunning the pipeline. + warned=$((warned + 1)) + echo "::warning::stale translation of a superseded docs version (latest is $LATEST_DOCS): $f" + echo " the pipeline removes superseded translations on its next run; not a blocker." + else + rc=1 + echo "::error::stale translation: $f" + echo " English source: $src" + echo " recorded digest: sha256:${actual}" + echo " current digest: sha256:${expected}" + echo " fix: refresh the translation, then run 'hack/check-i18n.sh update-digests'" + fi fi done < <(translated_digest_files) - [ "$rc" -eq 0 ] && echo "translation freshness: OK ($checked translated pages match their English source)" + if [ "$rc" -eq 0 ] && [ "$warned" -eq 0 ]; then + echo "translation freshness: OK ($checked translated pages match their English source)" + elif [ "$rc" -eq 0 ]; then + echo "translation freshness: no blocking drift ($checked translated pages checked, $warned advisory warning(s) above)" + fi return "$rc" } update_digests() { + # With explicit files, re-stamp exactly those — the escape hatch for a + # transcreation a human just refreshed. With no arguments, re-stamp every + # machine-translated page but SKIP hand-localized ones: their stale digest + # is the only signal that the transcreation drifted, and a wholesale + # re-stamp would silence it while the drift persists. local f - while IFS= read -r f; do + { + # `|| true`: with zero stamped translations the underlying grep exits 1, + # which under `set -euo pipefail` would silently kill the whole script. + if [ "$#" -gt 0 ]; then printf '%s\n' "$@"; else translated_digest_files || true; fi + } | while IFS= read -r f; do [ -z "$f" ] && continue + if [ "$#" -eq 0 ] && is_transcreate "$f"; then + echo "skip $f — hand-localized (l10n: transcreate); refresh it by hand, then: hack/check-i18n.sh update-digests $f" + continue + fi local src expected tmp src="$(en_source_for "$f")" if [ ! -f "$src" ]; then echo "::warning::skip $f — English source missing: $src" continue fi + # The awk below only REWRITES an existing line; without this check a page + # missing the field would pass through untouched yet be reported "updated". + if ! grep -qE '^source_digest:' "$f"; then + echo "::warning::skip $f — no source_digest line in the front matter; add one, then re-run" + continue + fi expected="$(sha256 "$src")" tmp="$(mktemp)" awk -v d="sha256:${expected}" ' @@ -159,7 +263,7 @@ update_digests() { ' "$f" > "$tmp" mv "$tmp" "$f" echo "updated $f -> sha256:${expected}" - done < <(translated_digest_files) + done } cmd="${1:-check}" @@ -171,10 +275,11 @@ case "$cmd" in exit "$rc" ;; update-digests) - update_digests + shift + update_digests "$@" ;; *) - echo "usage: $0 [check|update-digests]" >&2 + echo "usage: $0 [check|update-digests [file...]]" >&2 exit 2 ;; esac diff --git a/hack/i18n/.gitignore b/hack/i18n/.gitignore new file mode 100644 index 000000000..6702024fe --- /dev/null +++ b/hack/i18n/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.pyc +# venv bootstrapped by run-daily.sh (repo root); listed here too for safety +.venv-i18n/ diff --git a/hack/i18n/README.md b/hack/i18n/README.md new file mode 100644 index 000000000..38a7c8a82 --- /dev/null +++ b/hack/i18n/README.md @@ -0,0 +1,325 @@ +# Cozystack website translation pipeline + +Automated, in-tree localization for cozystack.io with a machine review gate. +Detects English pages that are new or changed and, for each one, runs a +translate → verify → review → revise loop. Runs daily on a Claude subscription +until the usage limit is hit; a maintainer merges one translation PR per week. + +Lives in the website repo (vendor-neutral, community-inspectable) alongside the +existing `hack/check-i18n.sh` CI guard. + +## Per-page pipeline (every page, all languages) + +``` +content/en/** ──▶ worklist.py (missing | stale, via source_digest) + │ + ▼ + translate (Opus + glossary + style guide) + │ + ▼ + back-translate ─▶ compare vs original (meaning-drift check) + │ + ▼ + deterministic checks (version/flag/do-not-translate integrity, + per-language typography — lib.py, no model calls) + │ + ▼ + review ×2 (native speakers of the target language) + • technical editor → fluency / register / terminology + • Cozystack maintainer → technical correctness vs source + │ + findings? ──yes──▶ revise ──▶ re-check (≤ max_rounds) + │ no │ rounds exhausted, + │ │ findings still open + ▼ ▼ + write content//** write content//** anyway + stamped `auto-reviewed` stamped `auto-reviewed-with-findings` + │ │ + └──────────────┬──────────────┘ + ▼ + check-i18n.sh → weekly PR → merged by a maintainer +``` + +The gate **triages, it does not block**: a page that clears it is stamped +`auto-reviewed`; one that exhausts the revise rounds with findings still open +is published anyway, stamped `auto-reviewed-with-findings`, and its findings +are posted to the weekly PR — those pages are the ones worth a human's +attention first. What the gate does guarantee is *atomicity*: nothing is +written until the review finishes, so an interrupted daily run never leaves a +half-done page, and the text written is always exactly the text the reviewers +last saw. Native human owners flip either stamp to `ratified` after their pass. + +## Auth — Max subscription via the Claude Agent SDK + +The pipeline runs on the maintainer's Claude subscription (Max), so there is no +per-token API billing. Note: the **base `anthropic` SDK cannot use a +subscription** (metered API only), so `translate.py` calls the **Claude Agent +SDK** (`claude-agent-sdk`), which reads `CLAUDE_CODE_OAUTH_TOKEN`. Set it up once +on the machine that runs the daily job: + +```bash +npm install -g @anthropic-ai/claude-code # if the `claude` CLI isn't installed +claude setup-token # prints sk-ant-oat01-... (~1-year token) +export CLAUDE_CODE_OAUTH_TOKEN=sk-ant-oat01-... +unset ANTHROPIC_API_KEY # it would shadow the subscription and bill metered API +``` + +The run hard-fails if `ANTHROPIC_API_KEY` is set. Cost is not the constraint — +the daily subscription usage limit is; the run stops cleanly at the limit and +resumes the next day. + +Verify the active credential before running: + +```bash +[ -n "$ANTHROPIC_API_KEY" ] && echo "metered API (wrong)" || \ +{ [ -n "$CLAUDE_CODE_OAUTH_TOKEN" ] && echo "subscription (ok)" || echo "no credential"; } +``` + +## Cadence — daily run, weekly PR + +```bash +hack/i18n/run-daily.sh # all languages, until the limit +hack/i18n/run-daily.sh --lang ru # one language +``` + +Schedule it once a day (cron/launchd). Each run translates and review-gates as +many pages as the day's quota allows, stops cleanly at the limit, and commits +onto a single `i18n/week-` branch. One PR per week is opened and kept +updated; **a maintainer merges it**. Running daily uses each day's quota (a +weekly-only run would use ~1/7 of the available capacity), while the weekly PR +keeps review to one deliberate merge instead of a stream of them. + +Over time this clears the backlog, then just keeps translations fresh against +the English source. + +### Capture the output — two conditions report ONLY to stdout/stderr + +The daily-quota stop ("usage limit reached — stopping cleanly") and a page +skipped after repeated protocol errors both surface only in the run's output: +there is no GitHub Action and no other durable channel. If cron/launchd +discards output, a page that fails every single day is invisible. Point the +job at an append-only log: + +```bash +0 9 * * * cd ~/cozystack-website-i18n && ./hack/i18n/run-daily.sh >> ~/i18n-run.log 2>&1 +``` + +and glance at two health signals weekly: `grep -c '::warning::skipped' ~/i18n-run.log` +(a page stuck on protocol errors) and `python3 hack/i18n/worklist.py | head -3` +(the backlog must trend to zero, then hover near it). A second invocation while +one is still running exits immediately — the script takes a per-clone pidfile +lock in `$TMPDIR`, so an overlapping manual run cannot race a slow cron run. + +Manual/dry inspection: + +```bash +python3 hack/i18n/worklist.py # what needs work +python3 hack/i18n/translate.py --dry-run # plan, no model calls +./hack/check-i18n.sh # freshness + i18n key parity +``` + +### Throughput, measured + +The backlog is **183 pages × 6 configured languages ≈ 1100 jobs**. Each job is +5–15 Opus calls (translate, back-translate, compare, two reviewers, and a +revise round per finding), so the backlog is on the order of 5k–16k model calls. + +Measured pilots on a subscription, both on the same ~2500-word release blog post: + +| Language | Wall clock | Outcome | +|----------|-----------|---------| +| de | 2m54s | cleared the gate on the first round | +| ru | 11m24s | went through the revise loop, findings still open | + +So per-page cost is dominated by **whether the revise loop runs**, not by the +page alone — a clean page is ~3 min, one that keeps failing review is ~4× that. +Taking the range against ~1100 jobs puts the backlog at roughly 55–220 hours of +wall clock, before daily usage limits enter the picture at all. That is weeks to +months on a personal subscription, which is why the plan is to bootstrap the +backlog with `auth: api-key` on an organization account and leave the +daily/weekly cadence to handle only steady-state drift afterwards. + +Most docs pages are far shorter than a release blog post, so treat these as an +upper bound rather than an average. + +### Placeholder-dense pages + +Code, shortcodes, and inline code are swapped for opaque `§§…§§` placeholders +before translation and restored after (see `protect`/`restore` in `lib.py`). A +long, code-heavy page can carry 60+ of them, and a model occasionally drops one. +The gate refuses to write a page with a dropped or duplicated placeholder — a +silently deleted code block is worse than a retry — so `translate.py` retries +such a page a few times within the run (the loss is non-deterministic and +usually clears). A page that still fails every attempt is skipped and stays in +the worklist; it is not stalled on, but it also will not publish until an attempt +succeeds. A page that reproducibly fails is a signal to translate it by hand. + +## Files + +| Path | Purpose | +|------|---------| +| `config.yaml` | languages, scope, blog cutoff, auth mode, back-translation + review config | +| `requirements.txt` | pinned Python deps — the runner installs on every invocation, so versions must not float | +| `glossary.yaml` | do-not-translate terms + preferred per-language terms | +| `prompts/translate.md` | translation system prompt | +| `prompts/back-translate.md` / `back-translate-compare.md` | round-trip meaning-drift check | +| `prompts/review-editor.md` | native technical-editor reviewer (fluency) | +| `prompts/review-maintainer.md` | native Cozystack-maintainer reviewer (technical correctness) | +| `prompts/revise.md` | revise a translation against reviewer findings | +| `style-guides/.md` | per-language tone/register conventions | +| `lib.py` | shared helpers (config, discovery, digest, front matter, protect/restore, integrity + typography checks) | +| `lint_translations.py` | typography lint for already-published translations (advisory; `--strict` to gate) | +| `worklist.py` | diff detector | +| `translate.py` | translate + review-gate driver; also removes orphaned translations | +| `run-daily.sh` | daily runner (subscription, until limit, commit + PR) | +| `test_i18n.py` | tests for the pure functions (no network) — `python3 hack/i18n/test_i18n.py` | +| `../../layouts/partials/translation-banner.html` | reader-facing machine-translation disclaimer | + +## Scope + +- **Docs:** latest version only; older versions stay English (they are `noindex`). The version-picker landing (`docs/_index.md`) is version-agnostic and always in scope. +- **Blog:** posts newer than `blog_since` in `config.yaml` — normally a rolling window (`"60d"` = today − 60 days, resolved per run), so the cutoff doesn't rot as the site ages. Aged-out posts keep their existing translations; they just stop being refreshed. +- **Never:** `docs/next/**`, `**/_include/**`. +- **Deleted English pages:** their translations are removed on the next run (only `source_digest`-stamped, i.e. pipeline-managed files — hand-authored locale-only pages are never touched), so the weekly PR carries the deletion. A mass-deletion floor refuses to remove translations of more than 5 distinct English pages at once — a large batch means the English tree moved, not that a handful of pages died. Known limitation: only `.md`/`.html` files are removed; images of a deleted page bundle stay behind. +- **Hand-written pages (`l10n: transcreate`)** are never re-translated, even once their English source moves. Overwriting one would replace a person's work with literal machine output and clobber the `transcreate` marker in the same write, so the loss would be silent and unrepeatable. The run reports them in the weekly PR under *Hand-written pages skipped* instead; rework them by hand if the English change matters, then refresh `source_digest`. +- Code, shortcodes, comments, inline code, raw HTML tags, URLs, CLI, YAML keys, and brand names are preserved structurally or via the glossary. HTML tags are masked with their visible `alt`/`title`/`caption` values left exposed, so markup survives byte-for-byte while the text a reader sees is still translated. + +## Front-matter fields + +| Field | Meaning | Read by | +|-------|---------|---------| +| `source_digest` | sha256 of the English source the translation was made from — the freshness signal | `hack/check-i18n.sh`, `worklist.py` | +| `l10n` | *how* the page was localized: `mt` (machine) or `transcreate` | `build_worklist`, `translate.py`, `hack/check-i18n.sh`, plus humans triaging review | +| `translation_review` | *ratification state*: `auto-reviewed`, `auto-reviewed-with-findings`, or `ratified` | `translation-banner.html` | + +`l10n: transcreate` is load-bearing for the pipeline, not just a review hint: +mark a page `transcreate` → `worklist.py` skips it and `translate.py` refuses to +overwrite it → `check-i18n.sh` downgrades its source drift to a warning instead +of failing CI → you refresh the wording by hand → run `update-digests ` to +re-stamp `source_digest` so the warning clears. + +`translation_status: current` appears on pages from the i18n PoC. It is not +written or read by anything — `source_digest` computes freshness properly — so +this pipeline does not write it. + +## Adding a language + +Two lists, two different questions: + +| | `languages:` in `hack/i18n/config.yaml` | `languages:` in `hugo.yaml` | +|---|---|---| +| Answers | what gets **translated** | what gets **built and served** | +| Add it | as soon as you want the language | only once its content exists | + +They are allowed to differ in exactly one direction. **Translated but not +declared** is the normal way a language starts — the pipeline fills +`content//` while the site stays quiet about it. **Declared but not +translated** is broken, and `test_i18n.py` fails on it. + +Order matters, because declaring a language in `hugo.yaml` before +`content//` exists does not build nothing: Hugo emits `//`, +`//tags/`, `//categories/`, `//topics/`, +`//article_types/` and `//404.html` anyway, all `index, follow` and +self-canonical — roughly six empty but indexable pages per language. + +So the sequence is: add to `config.yaml` → let the pipeline translate → add to +`hugo.yaml` in the PR that lands the content. + +`es` and `pt-br` are at step one right now: they are translated, have their +`i18n/*.toml` ready, and are commented out in `hugo.yaml` until their content +lands. + +## Reader-facing disclosure + +`layouts/partials/translation-banner.html` puts a machine-translation notice on +every localized page and links to the English original. The rule is fail-safe: +the banner shows on any non-English page **unless** its front matter says +`translation_review: ratified`. Pages from the i18n PoC carry no +`translation_review` at all, and they are machine output too — keying off the +field's presence would silently exempt exactly the pages that need the notice. + +Localized pages are deliberately indexed from day one (no `noindex`): readers +get the docs now rather than after a native review that may take months. The +banner is what makes that trade honest. + +The banner is wired into **`docs/baseof.html` only** — documentation pages, by +product decision. It is deliberately absent from the blog, marketing/`page` +layouts, `resources`, and the homepage: the notice belongs where a wrong +technical detail is costly (an operator running a translated command), not +across a marketing hero. To change coverage, add or remove the +`{{ partial "translation-banner.html" . }}` call in the relevant layout; the +partial's own guard (non-English and not `ratified`) is layout-agnostic. + +## Rollback / kill switch + +Ordered from cheapest to most drastic. + +**Stop translating** — remove the cron entry, or just stop running +`run-daily.sh`. Nothing else runs on its own; there is no GitHub Action. + +**Stop a single language** — comment it out of `languages:` in `config.yaml`. +Existing pages stay served. + +**Take a language out of the site without deleting it** — comment its block out +of `languages:` in `hugo.yaml`. Hugo stops building `//` entirely; the +`content//` tree stays in git and can be re-enabled with one edit. This is +the right move if a language's quality is disputed: it de-indexes without losing +the work. + +**De-index one page but keep it served** — add `robots: "noindex, follow"` to its +front matter and have `layouts/partials/hooks/head-end.html` honour it, or delete +the page (its English source is untouched, and `worklist.py` will simply see it +as missing again). + +**Undo a bad run** — every run lands on one `i18n/week-` branch and is +merged by a maintainer, so the revert is a normal `git revert` of the merge +commit. Nothing reaches production without that merge. + +**Discard a week entirely** — close the PR **and delete the branch**. The runner +resumes on branch existence, not PR state: a surviving branch of a +closed-without-merge PR would become the base of the next week and carry the +rejected translations forward. + +**Full stop** — revert the merge, comment the non-English languages out of +`hugo.yaml`, and the site is English-only again. The pipeline touches nothing but +`content//`. + +## Governance + +Nothing here changes CODEOWNERS or branch protection. The pipeline only ever +opens a PR touching `content//`; a maintainer reviews and merges it +weekly. There is deliberately **no auto-merge code path** — "nothing reaches +the production site without a maintainer merging it" is a property of the +runner, not a config default. Code, layouts, config, the English source, and +this pipeline itself remain maintainer-owned as before. + +Pages are stamped `translation_review: auto-reviewed` when they clear the +machine gate and `auto-reviewed-with-findings` when the revise loop ran out of +rounds with findings still open — the latter are the ones worth a human's +attention first. Native owners flip either to `ratified` after their pass. + +**What the gate is and is not.** The two "native reviewers" are the same model as +the translator, given different system prompts. That measures self-consistency +and catches the obvious failures (meaning drift, mangled terminology, broken +register); it is **not** native-speaker ratification, and `auto-reviewed` must +not be read as "a human checked this". Nor does it ever block publication — it +is a triage classifier, not a gatekeeper: every page that survives the +placeholder checks is published, and the stamp says how it fared. Only a human +sets `ratified`, and only `ratified` removes the reader-facing disclaimer +banner. The gate's real job is to +make the machine output good enough to publish while native review happens +asynchronously — not to replace it. + +## Ownership and continuity + +The pipeline currently authenticates with a maintainer's Claude subscription +(`auth: oauth-subscription`) to bootstrap the backlog. The intent is to move to +an organization-owned API key (`auth: api-key` + `ANTHROPIC_API_KEY`) once it is +provisioned, so the pipeline does not depend on one individual's account. That +switch is a config change; no code changes. + +## Secrets + +- Translation auth: `CLAUDE_CODE_OAUTH_TOKEN` (from `claude setup-token`) — subscription, not metered. Only needed headless; if the `claude` CLI is logged in on the runner, the Agent SDK picks that credential up. +- Under `auth: oauth-subscription`, never set `ANTHROPIC_API_KEY` — it shadows the subscription and silently bills metered API (the run hard-fails if it's present). +- There is no GitHub Action and no CI-held credential: the pipeline runs from a maintainer's clone, by hand or by cron. That is deliberate — it keeps a personal subscription token out of repo secrets. diff --git a/hack/i18n/config.yaml b/hack/i18n/config.yaml new file mode 100644 index 000000000..e4c57cb8f --- /dev/null +++ b/hack/i18n/config.yaml @@ -0,0 +1,167 @@ +# Cozystack website translation pipeline — configuration. +# +# Single source of truth for the automated i18n pipeline: which languages +# exist, which model translates which content, and which paths are in scope. +# Consumed by hack/i18n/*.py and hack/i18n/run-daily.sh. + +source_lang: en +content_dir: content + +# Auth. Two supported modes — pick one; switching is a config change only, the +# code path is identical: +# oauth-subscription : a Claude subscription (Max) via the Claude Agent SDK. +# No metered billing. Credential = a logged-in `claude` +# CLI on the runner, or CLAUDE_CODE_OAUTH_TOKEN +# (`claude setup-token`) for headless/CI. The run refuses +# to start if ANTHROPIC_API_KEY is set, since that would +# shadow the subscription and bill metered API. +# api-key : metered Claude API billed to an organization account. +# Requires ANTHROPIC_API_KEY. Prefer an org-owned key +# (not a personal one) so the pipeline does not depend on +# one individual. +# +# Current plan: bootstrap the backlog on a maintainer's subscription, then move +# to an organization-owned API key once it is provisioned — see README +# "Ownership and continuity". +auth: oauth-subscription + +# How translated output reaches the site. Only `pr_only` is implemented: the +# runner pushes to a weekly branch and opens a PR for a maintainer to merge — +# nothing reaches production without that merge, and CODEOWNERS / branch +# protection are untouched. There is deliberately no auto-merge mode; run-daily.sh +# fails closed on any other value rather than silently doing something unsafe. +publish_mode: pr_only + +# Cadence: run daily until the account's usage limit is hit. On a usage-limit +# error the pipeline stops cleanly (exit 0) and resumes the next day. Pages are +# written only after the review gate finishes, so an interrupted run never +# leaves a half-processed page. + +# Target languages. `code` must match the Hugo `languages:` key in hugo.yaml +# and the i18n/.toml filename. `hreflang` is the BCP-47 tag emitted in +# . +# +# This list is what gets TRANSLATED. `languages:` in hugo.yaml is what gets +# BUILT AND SERVED. They are deliberately allowed to differ, in one direction: +# +# here but not in hugo.yaml = being translated, not shipped yet. Correct and +# expected — this is how a new language starts. +# in hugo.yaml but not here = shipped but never translated or kept fresh. +# Broken; hack/i18n/test_i18n.py fails on it. +# +# A language is declared in hugo.yaml in the same PR that lands its first +# content, never before: Hugo builds //, //tags/, //categories/ +# and more for a declared language even with an empty content tree, all +# `index, follow` — publishing empty pages to the index. See hugo.yaml. +languages: + - code: de + name: Deutsch + hreflang: de + - code: ru + name: Русский + hreflang: ru + - code: zh-cn + name: 简体中文 + hreflang: zh-Hans + - code: hi + name: हिन्दी + hreflang: hi + # Translated now, not yet declared in hugo.yaml — they ship once their content + # lands (see "Adding a language" in README.md). + - code: es + name: Español + hreflang: es + - code: pt-br + name: Português (Brasil) + hreflang: pt-BR + +# Model. Every stage — translation, back-translation, both reviewers, revision — +# runs on Opus for maximum quality (decision 2026-07-17). Max output tokens and +# sampling are left at the Agent SDK defaults; the review gate, not a token cap, +# is what bounds quality here. +model: + default: claude-opus-4-8 + +# Scope. +# +# Docs: ONLY the latest version is translated — older versions are noindex, so +# translating them burns the daily budget on pages search engines ignore. The +# version is NOT hardcoded here: lib.py reads `params.latest_version_id` from +# hugo.yaml, so a release bump moves the pipeline automatically and no stale +# version list has to be maintained. Everything under docs/ that is not the +# latest version (including `next`) is skipped. +# +# Blog: only recent posts are worth translating × every language. `blog_since` +# drops any blog post whose date-prefixed path (blog/YYYY-MM-DD-…) is older +# than the cutoff. "d" is a rolling window (today − N days, resolved per +# run) — the normal setting, because a fixed ISO date quietly rots into "no +# blog post is ever in scope". A fixed date ("2026-05-17") and "" (entire +# archive) are also accepted. Already-translated posts that age out are NOT +# deleted; they just stop being refreshed. +blog_since: "60d" +# Patterns are matched with pathlib semantics (right-anchored): a bare "*.md" +# means "any markdown page at any depth", "docs/**" means the docs subtree. +# Effective scope = these globs, minus exclude_globs, minus non-latest docs +# versions, minus blog posts older than blog_since. +translate_globs: + - "*.md" + - "docs/**" # narrowed to the latest version by lib.py (see above) + - "blog/**" +exclude_globs: + - "**/_include/**" + # oss-health pages are shells: front matter plus `layout: oss-health-app`, + # which renders a dashboard client-side from live English data (devstats, + # OpenSSF, telemetry). Translating them localizes the page chrome around a + # dashboard that stays English — a half-translated page — and they are + # high-churn, low-search-value. They also carry the only `lede` front-matter + # field on the site; excluding them is what keeps the translatable-key list + # honest (see TRANSLATABLE_FRONTMATTER_KEYS in lib.py). Revisit if the + # dashboards ever get localized data. + - "oss-health/**" + +# Quality gate — runs on EVERY page (not just hero) before it is written. +# Each page goes through: translate -> back-translate -> two native reviewers +# -> revise-if-needed. The gate never blocks publication — it triages: a page +# that clears it is stamped `auto-reviewed`, one that exhausts the revise +# rounds with findings still open is written anyway and stamped +# `auto-reviewed-with-findings` (review those first). Nothing is written until +# the gate FINISHES, so an interrupted run never leaves a half-processed page. +# All stages use Opus via the OAuth subscription. +back_translation: + enabled: true + # Round-trip check: translate the localized page back to English and compare + # to the original. If meaning drifted (added/dropped/changed claims, numbers, + # or technical facts), the page is revised and re-checked. max_retries bounds + # the revise loop per page. + max_retries: 2 + +review: + # Two virtual native reviewers per target language, each with a distinct remit. + # Both must pass (or the page is revised addressing their findings and + # re-reviewed, up to max_rounds). Reviewers are prompted AS native speakers of + # the target language. + max_rounds: 2 + reviewers: + - id: technical-editor + prompt: prompts/review-editor.md # literary quality / fluency / register + - id: cozystack-maintainer + prompt: prompts/review-maintainer.md # technical correctness (k8s/Cozystack) + +# Front-matter stamp written on pages that clear the machine gate. Native +# human owners flip this to "ratified" after their pass; the site can badge or +# prioritize by it. Machine gate never sets "ratified". +review_status_field: translation_review +review_status_value: auto-reviewed + +# Publishing: the runner always opens/updates a weekly PR and stops — a +# maintainer merges it. There is deliberately NO auto-merge option: "no machine +# output reaches the production site without a maintainer merging it" is meant +# as a property of the pipeline, not a default. Nothing about CODEOWNERS or +# branch protection changes. The runner translates DAILY (to use each day's +# quota) but accumulates into ONE branch per ISO week, so maintainers review +# and merge a single translation PR per week instead of a stream of them. + +# SEO note: there is deliberately no keyword-research step here. The English +# source is already SEO-optimized during authoring, so the translation carries +# that intent over — the translate prompt transcreates title/description to read +# naturally in the target language rather than translating them literally. diff --git a/hack/i18n/glossary.yaml b/hack/i18n/glossary.yaml new file mode 100644 index 000000000..a9cd34f67 --- /dev/null +++ b/hack/i18n/glossary.yaml @@ -0,0 +1,138 @@ +# Cozystack translation glossary. +# +# Two lists: +# do_not_translate — terms kept verbatim in every language (brands, product +# names, protocols, CLIs, APIs). The translator is instructed to never +# translate, transliterate, or inflect these. +# preferred — for terms that DO get translated, the canonical rendering per +# language, so translations stay consistent across pages. Derived from the +# CNCF Cloud Native Glossary and kubernetes.io localization teams. +# +# THE LISTS ARE CASE-SENSITIVE, and one term appears in both on purpose: +# `Tenant` (do_not_translate) — the Cozystack API kind. A reader has to be +# able to match it to `kind: Tenant` in a manifest, so it stays verbatim. +# `tenant` (preferred) — the ordinary noun ("a tenant's workloads"). +# Leaving this in English makes the prose unreadable, so it gets translated. +# This is a real distinction, not a mistake — keep the capitalization exact if +# you edit either entry. +# +# Code blocks, inline `code`, Hugo shortcodes ({{< >}} / {{% %}}), URLs, and +# front-matter keys are protected structurally by the pipeline (see lib.py), +# not via this list. + +do_not_translate: + # Project / brand + - Cozystack + - Ænix + - Aenix + # CNCF ecosystem + - Kubernetes + - k8s + - KubeVirt + - Cilium + - LINSTOR + - Talos + - Talos Linux + - FluxCD + - Flux + - Argo CD + - VictoriaMetrics + - VictoriaLogs + - Helm + - Prometheus + - Grafana + - etcd + - containerd + - CNCF + # Vendors / external + - VMware + - vSphere + - Proxmox + - OpenStack + - AWS + - GCP + - Azure + - Broadcom + # Protocols / specs / formats + - CSI + - CNI + - CRI + - OCI + - API + - REST + - gRPC + - YAML + - JSON + - TLS + - DNS + - SBOM + - CSAF + - VEX + - SLSA + # Cozystack nouns kept as-is + - Package + - PackageSource + - ApplicationDefinition + - Tenant + - SchedulingClass + +# Preferred translations for translatable terms. Keys are the English term; +# values are per-language canonical renderings. Omitted languages fall back to +# the translator's judgement (still consistent within a run via glossary hint). +preferred: + cluster: + ru: кластер + de: Cluster + es: clúster + pt-br: cluster + zh-cn: 集群 + hi: क्लस्टर + node: + ru: узел + de: Knoten + es: nodo + pt-br: nó + zh-cn: 节点 + hi: नोड + storage: + ru: хранилище + de: Speicher + es: almacenamiento + pt-br: armazenamento + zh-cn: 存储 + hi: स्टोरेज + workload: + ru: рабочая нагрузка + de: Workload + es: carga de trabajo + pt-br: carga de trabalho + zh-cn: 工作负载 + hi: वर्कलोड + deployment: + ru: развёртывание + de: Deployment + es: despliegue + pt-br: implantação + zh-cn: 部署 + hi: डिप्लॉयमेंट + virtual machine: + ru: виртуальная машина + de: virtuelle Maschine + es: máquina virtual + pt-br: máquina virtual + zh-cn: 虚拟机 + hi: वर्चुअल मशीन + bare metal: + ru: bare-metal + de: Bare-Metal + es: bare metal + pt-br: bare metal + zh-cn: 裸金属 + hi: बेयर मेटल + tenant: + ru: тенант + de: Mandant + es: inquilino (tenant) + pt-br: tenant + zh-cn: 租户 + hi: टेनेंट diff --git a/hack/i18n/lib.py b/hack/i18n/lib.py new file mode 100644 index 000000000..0fb99f173 --- /dev/null +++ b/hack/i18n/lib.py @@ -0,0 +1,774 @@ +"""Shared helpers for the Cozystack website translation pipeline. + +Pure-stdlib except PyYAML. No Hugo/Node required. Everything keys off the same +`source_digest` (sha256 of the English source) convention that hack/check-i18n.sh +already enforces, so the automated pipeline and the CI lint agree by construction. +""" + +from __future__ import annotations + +import copy +import datetime +import hashlib +import os +import re +from dataclasses import dataclass +from pathlib import PurePosixPath + +import yaml + +REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +CONFIG_PATH = os.path.join(os.path.dirname(__file__), "config.yaml") +GLOSSARY_PATH = os.path.join(os.path.dirname(__file__), "glossary.yaml") + +# Front-matter values we translate, AT ANY DEPTH. The landing page keeps +# user-visible copy in nested structures (taglines[], benefits[].title, +# features[].description, seo.title …) which Hugo renders into the hero and the +# cards — translating only top-level keys would copy the English hero verbatim +# into every locale and silently revert hand-translated pages. +# Everything else (slug, date, weight, aliases, images, icon, source_digest, …) +# is preserved verbatim. +TRANSLATABLE_FRONTMATTER_KEYS = ( + "title", "linkTitle", "description", "summary", + # homepage use_cases[] cards (main's home use-cases feature): user-visible prose; + # `link` stays structural. Reached by the recursive walk into the use_cases list. + "situation", "outcome", "link_text", +) +# Keys whose value is a list of user-visible strings, each translated. +TRANSLATABLE_LIST_KEYS = ("taglines", "keywords") + +# Protected spans: replaced with opaque placeholders before the model sees the +# text, restored afterwards. Order matters (fenced code before inline code). +# SHORTCODE is handled specially (see protect) so that visible-text attributes +# inside it are still translated; the others are protected wholesale. +_FENCE_RE = re.compile(r"```.*?```", re.DOTALL) +_SHORTCODE_RE = re.compile(r"\{\{[<%].*?[%>]\}\}", re.DOTALL) +_HTMLCOMMENT_RE = re.compile(r"", re.DOTALL) +_INLINECODE_RE = re.compile(r"`[^`\n]+`") +# Raw HTML tags. `.html` pages are mostly markup, and Markdown pages embed it +# freely, so without this the model receives `
`, `
` and +# `` verbatim and is free to translate an attribute, drop a +# closing tag or mangle a URL. Hugo renders the damage without complaining and +# the digest check only compares the English source, so nothing downstream +# catches it. Matching the tag name conservatively (a letter, then name +# characters, then a boundary) keeps prose like `x ]*?)?/?>", re.DOTALL) + +# Shortcode attributes whose VALUE renders as visible text and must be +# translated: figure captions (shown under the image), alt text (screen readers +# + SEO), and titles. Everything else in a shortcode (src, width, id, class, +# link, the delimiters, the param names) is structure and stays verbatim. +_VISIBLE_ATTR_RE = re.compile(r'\b(?:caption|alt|title)="([^"]*)"') + +# Link DESTINATIONS. Only backticked code was structurally protected before, so a +# URL sitting in ordinary prose (`[text](https://…)`) was defended by a prompt rule +# alone — the model could silently mutate, localize or "fix" it and nothing would +# catch it. Masking the destination makes URL integrity a guarantee instead of a +# hope, while leaving the link TEXT exposed so it still gets translated. +# inline: [text](/docs/install "Optional title") -> destination only +# autolink: +# refdef: [id]: https://example.com +# Known ceiling: a destination with unescaped parens (`/wiki/Foo_(bar)`) masks +# only up to the first `)`, leaving a stray paren exposed to the model. Restore +# still round-trips; the risk is the model "tidying" the stray paren. No +# in-scope page has one; extend to balanced-paren matching if that changes. +_LINKDEST_RE = re.compile(r'(?<=\])\((?P<[^>]*>|[^)\s]*)(?P\s+"[^"]*")?\)') +_AUTOLINK_RE = re.compile(r'<(?:https?|mailto):[^>\s]+>') +# `(?!\^)` keeps footnote definitions (`[^1]: prose`) out: their "destination" +# is the first word of translatable prose, not a URL. +_REFDEF_RE = re.compile(r'(?m)^(?P<label>\[(?!\^)[^\]]+\]:[ \t]*)(?P<dest>\S+)') + + +def load_config() -> dict: + with open(CONFIG_PATH, encoding="utf-8") as fh: + return yaml.safe_load(fh) + + +def load_glossary() -> dict: + with open(GLOSSARY_PATH, encoding="utf-8") as fh: + return yaml.safe_load(fh) + + +def sha256_file(path: str) -> str: + """sha256 hex of raw file bytes — matches hack/check-i18n.sh.""" + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(65536), b""): + h.update(chunk) + return h.hexdigest() + + +def _matches_any(rel: str, globs: list[str]) -> bool: + """Match rel against glob patterns with path (not string) semantics. + + PurePosixPath.match is right-anchored and segment-aware: `*.md` matches the + BASENAME, so it matches a .md page at any depth (this is intended — see + `translate_globs` in config.yaml), while `docs/*` matches one level only + rather than silently swallowing the whole subtree the way fnmatch's `*` + does by matching `/` too. + + Breadth here is deliberate: `*.md` casts wide and scope is then narrowed by + exclude_globs, _docs_out_of_scope, and _blog_too_old. Those are the checks + that bound the work, not this one. + """ + p = PurePosixPath(rel) + for g in globs: + if p.match(g): + return True + # PurePosixPath.match has no `**` recursion; expand `a/**` to a prefix test. + if g.endswith("/**") and (rel == g[:-3] or rel.startswith(g[:-2])): + return True + if g.startswith("**/") and p.match(g[3:]): + return True + return False + + +def latest_docs_version(cfg: dict) -> str: + """Read `params.latest_version_id` from hugo.yaml — the single source of truth. + + FAILS CLOSED: if hugo.yaml or the key is missing/renamed we raise rather than + return None. Returning None would make _docs_out_of_scope a no-op and quietly + pull every old, noindex'd docs version (and `next`) into scope — 7× the work, + burning quota on pages search engines ignore, and nobody would notice for + weeks. Hardcoding the version here would drift for the same reason. + """ + path = os.path.join(REPO_ROOT, "hugo.yaml") + if not os.path.exists(path): + raise RuntimeError(f"hugo.yaml not found at {path}: cannot determine the latest " + f"docs version, refusing to guess the translation scope") + with open(path, encoding="utf-8") as fh: + hugo = yaml.safe_load(fh) or {} + latest = (hugo.get("params") or {}).get("latest_version_id") + if not latest: + raise RuntimeError("hugo.yaml has no params.latest_version_id: cannot determine the " + "latest docs version, refusing to guess the translation scope") + return str(latest) + + +def _docs_out_of_scope(rel: str, latest: str) -> bool: + """True for any docs page that is not in the latest version (incl. `next`). + + Pages that live directly under docs/ (the version-picker landing, + docs/_index.md) are not version-specific and stay in scope: they are + translated, indexed pages whose freshness hack/check-i18n.sh tracks, so the + pipeline must keep them fresh. Only the versioned subtrees (docs/<ver>/...) + are narrowed to the latest version. + """ + parts = rel.split("/") + if parts[0] != "docs": + return False + if len(parts) < 3: + return False # docs/<file> — version-agnostic, always in scope + return parts[1] != latest + + +_BLOG_DATE_RE = re.compile(r"^blog/(\d{4}-\d{2}-\d{2})-") + + +def _blog_too_old(rel: str, blog_since: str) -> bool: + """True if rel is a blog post dated before the blog_since cutoff. + + blog_since is either an ISO date ("2026-05-17") or a rolling window ("60d", + meaning today minus 60 days, resolved at scope-computation time). A fixed + date quietly rots into "no blog post is ever in scope" as the site ages; + the rolling form is what the config should normally use. + """ + if not blog_since: + return False + m = _BLOG_DATE_RE.match(rel) + if not m: + return False + w = re.fullmatch(r"(\d+)d", blog_since.strip()) + if w: + cutoff = (datetime.date.today() - datetime.timedelta(days=int(w.group(1)))).isoformat() + else: + cutoff = blog_since + return m.group(1) < cutoff + + +def iter_source_files(cfg: dict) -> list[str]: + """Relative paths (under content/en) in scope for translation.""" + src_root = os.path.join(REPO_ROOT, cfg["content_dir"], cfg["source_lang"]) + latest = latest_docs_version(cfg) + out: list[str] = [] + for dirpath, _dirs, files in os.walk(src_root): + for name in files: + if not name.endswith((".md", ".html")): + continue + rel = os.path.relpath(os.path.join(dirpath, name), src_root) + rel = rel.replace(os.sep, "/") + if cfg.get("exclude_globs") and _matches_any(rel, cfg["exclude_globs"]): + continue + if _blog_too_old(rel, cfg.get("blog_since", "")): + continue + if _docs_out_of_scope(rel, latest): + continue + if _matches_any(rel, cfg["translate_globs"]): + out.append(rel) + return sorted(out) + + +def source_path(cfg: dict, rel: str) -> str: + return os.path.join(REPO_ROOT, cfg["content_dir"], cfg["source_lang"], rel) + + +def target_path(cfg: dict, lang: str, rel: str) -> str: + return os.path.join(REPO_ROOT, cfg["content_dir"], lang, rel) + + +@dataclass +class WorkItem: + lang: str + rel: str + reason: str # "missing" | "stale" + + +def recorded_digest(path: str) -> str | None: + """Read source_digest from a translated file's front matter, if present.""" + if not os.path.exists(path): + return None + # Read the whole front matter, not a fixed prefix: a long front matter would + # push source_digest past a 4096-byte cut, the page would look permanently + # stale, and it would be re-translated every single day. + with open(path, encoding="utf-8") as fh: + head = fh.read() + fm_end = head.find("\n---", 4) + if fm_end != -1: + head = head[:fm_end] + m = re.search(r"^source_digest:\s*\"?sha256:([0-9a-fA-F]+)\"?", head, re.MULTILINE) + return m.group(1) if m else None + + +_L10N_RE = re.compile(r"^l10n:\s*\"?([A-Za-z-]+)\"?", re.MULTILINE) + + +def l10n_mode(path: str) -> str | None: + """Read the `l10n` front-matter field of a translated page, if present. + + `transcreate` marks a page a human wrote or reworked by hand, as opposed to + `mt` which the pipeline stamps on its own output. + """ + if not os.path.exists(path): + return None + with open(path, encoding="utf-8") as fh: + text = fh.read() + # Parse the front matter as YAML rather than regex-matching the raw text: a + # regex that only accepts the bare and double-quoted forms misses the equally + # valid single-quoted `l10n: 'transcreate'`, returns None, and the page is + # re-translated — exactly the hand-authored work this field is meant to guard. + fm, _body, _raw = split_frontmatter(text) + if not fm: + return None + v = fm.get("l10n") + return str(v) if v is not None else None + + +def find_transcreated(cfg: dict, only_lang: str | None = None) -> list[tuple[str, str]]: + """(lang, rel) pairs the pipeline must not overwrite, and why they came up. + + Only pages that the worklist would otherwise have picked up — a hand-written + page whose English source has not moved needs no reporting. + """ + out: list[tuple[str, str]] = [] + langs = [l["code"] for l in cfg["languages"] if only_lang in (None, l["code"])] + for rel in iter_source_files(cfg): + cur = sha256_file(source_path(cfg, rel)) + for lang in langs: + tp = target_path(cfg, lang, rel) + if (os.path.exists(tp) and l10n_mode(tp) == "transcreate" + and recorded_digest(tp) != cur): + out.append((lang, rel)) + return sorted(out) + + +def build_worklist(cfg: dict, only_lang: str | None = None) -> list[WorkItem]: + items: list[WorkItem] = [] + langs = [l["code"] for l in cfg["languages"] if only_lang in (None, l["code"])] + for rel in iter_source_files(cfg): + cur = sha256_file(source_path(cfg, rel)) + for lang in langs: + tp = target_path(cfg, lang, rel) + if not os.path.exists(tp): + items.append(WorkItem(lang, rel, "missing")) + elif recorded_digest(tp) != cur: + # A page marked `l10n: transcreate` was written or reworked by a + # human. Re-translating it replaces that work with literal + # machine output AND overwrites the marker with `mt`, so the + # loss is silent and unrepeatable — the signal that the page was + # hand-made is the first thing destroyed. Leave it alone and let + # the weekly PR report it, so a human decides whether the + # English change is worth reworking the translation for. + if l10n_mode(tp) == "transcreate": + continue + items.append(WorkItem(lang, rel, "stale")) + return items + + +def find_orphan_translations(cfg: dict, only_lang: str | None = None) -> list[str]: + """Translated pages whose English source no longer exists. + + Deleting an English page used to leave its translations behind forever: + the worklist iterates English sources, so nothing ever revisited the + leftovers, and check-i18n.sh could only report them. The pipeline removes + them instead (translate.py), so the weekly PR carries the deletion. + + Only files carrying a `source_digest` stamp are considered — the stamp is + what marks a page as pipeline-managed. Hand-authored locale-only pages + (no stamp) are never touched. + + A stamped translation is an orphan in two cases: its English source no + longer exists, or it is a superseded docs version — a `docs/<ver>/...` page + whose version is not the latest. Superseded versions are noindex on the + English side and never enter the worklist, so a stamped translation of an + old version is refreshed by nothing and removed by nothing; it drifts from + English forever and is unreachable from navigation. Removing it is what + keeps `content/<lang>/docs/v1.4/...` from rotting after the site moves to + v1.6. + + Aged-out blog posts are deliberately NOT treated as orphans: the English + post stays as an indexed historical record, and its existing translations + are kept (frozen, and shown with the not-ratified banner) rather than + deleted, so a reader who follows an old link still gets a translation. + """ + latest = latest_docs_version(cfg) + orphans: list[str] = [] + for lang in (l["code"] for l in cfg["languages"] if only_lang in (None, l["code"])): + lang_root = os.path.join(REPO_ROOT, cfg["content_dir"], lang) + for dirpath, _dirs, files in os.walk(lang_root): + for name in files: + if not name.endswith((".md", ".html")): + continue + tp = os.path.join(dirpath, name) + if recorded_digest(tp) is None: + continue + rel = os.path.relpath(tp, lang_root).replace(os.sep, "/") + if not os.path.exists(source_path(cfg, rel)) or _docs_out_of_scope(rel, latest): + orphans.append(tp) + return sorted(orphans) + + +# ---- front matter ----------------------------------------------------------- + +_FM_RE = re.compile(r"^---\n(.*?)\n---\n(.*)$", re.DOTALL) + + +def split_frontmatter(text: str) -> tuple[dict | None, str, str]: + """Return (frontmatter_dict, body, raw_frontmatter). dict is None if absent. + + Only YAML front matter (--- fences) is handled; Cozystack content uses YAML. + """ + m = _FM_RE.match(text) + if not m: + return None, text, "" + raw = m.group(1) + try: + fm = yaml.safe_load(raw) or {} + except yaml.YAMLError: + fm = None + return fm, m.group(2), raw + + +# ---- nested front-matter extraction ---------------------------------------- + + +def extract_translatable(fm: dict) -> dict[str, str]: + """Walk front matter and return {path: text} for every translatable string. + + Paths look like "title", "taglines[0]", "benefits[1].description", "seo.title" + — stable addresses we can send to the model and apply back without touching + anything structural (icons, dates, slugs, weights). + """ + out: dict[str, str] = {} + + def walk(node, path: str, key: str | None): + if isinstance(node, dict): + for k, v in node.items(): + walk(v, f"{path}.{k}" if path else k, k) + elif isinstance(node, list): + for i, v in enumerate(node): + if isinstance(v, str): + if key in TRANSLATABLE_LIST_KEYS and v.strip(): + out[f"{path}[{i}]"] = v + else: + walk(v, f"{path}[{i}]", key) + elif isinstance(node, str): + if key in TRANSLATABLE_FRONTMATTER_KEYS and node.strip(): + out[path] = node + + walk(fm, "", None) + return out + + +_PATH_RE = re.compile(r"([^.\[\]]+)|\[(\d+)\]") + + +def apply_translations(fm: dict, values: dict[str, str]) -> dict: + """Return a deep copy of fm with `values` ({path: text}) applied by path.""" + out = copy.deepcopy(fm) + for path, text in values.items(): + tokens = [(m.group(1), m.group(2)) for m in _PATH_RE.finditer(path)] + node = out + for i, (name, idx) in enumerate(tokens): + last = i == len(tokens) - 1 + key = name if name is not None else int(idx) + try: + if last: + node[key] = text + else: + node = node[key] + except (KeyError, IndexError, TypeError): + break # path no longer exists (source changed) — skip it + return out + + +def merge_target_only_keys(out_fm: dict, existing_fm: dict | None) -> dict: + """Preserve keys a human added to the translated page but that do not exist + in the English source (e.g. a locale-specific `seo:` block, `l10n:` notes). + + Without this, regenerating a page from the English front matter silently + deletes work native reviewers did by hand. + """ + if not existing_fm: + return out_fm + for k, v in existing_fm.items(): + if k not in out_fm: + out_fm[k] = v + return out_fm + + +# ---- protect / restore ------------------------------------------------------ + + +def protect(text: str) -> tuple[str, dict[str, str]]: + """Replace code/shortcodes/comments with placeholders. Returns (masked, map). + + Shortcodes are NOT protected wholesale: a `{{< figure >}}` carries a visible + `caption` (rendered under the image) and `alt` (screen readers, SEO). Masking + the whole shortcode left those in English on every localized page. Instead the + shortcode's structure is protected and its visible-text attribute values are + exposed between placeholders, so they translate like ordinary prose while + src/width/delimiters stay byte-for-byte. + """ + store: dict[str, str] = {} + counter = 0 + + def _stash(s: str, kind: str) -> str: + nonlocal counter + token = f"§§{kind}_{counter}§§" + store[token] = s + counter += 1 + return token + + def _sub(kind): + return lambda m: _stash(m.group(0), kind) + + # 1. Fenced code first (it may contain backticks/shortcode-like text). + text = _FENCE_RE.sub(_sub("FENCE"), text) + + # 2. Shortcodes: protect structure, expose caption/alt/title values. + def _shortcode(m: "re.Match") -> str: + sc = m.group(0) + out, pos = [], 0 + for a in _VISIBLE_ATTR_RE.finditer(sc): + out.append(_stash(sc[pos:a.start(1)], "SC")) # structure up to the value + out.append(a.group(1)) # visible text, left to translate + pos = a.end(1) + out.append(_stash(sc[pos:], "SC")) # trailing structure + return "".join(out) + text = _SHORTCODE_RE.sub(_shortcode, text) + + # 3. Comments, then inline code. + text = _HTMLCOMMENT_RE.sub(_sub("HTMLCOMMENT"), text) + text = _INLINECODE_RE.sub(_sub("INLINECODE"), text) + + # 4. Raw HTML tags. Visible attribute values are exposed between placeholders + # exactly as they are for shortcodes: masking a tag wholesale would leave alt + # and title text in English on every localized page. + def _htmltag(m: "re.Match") -> str: + tag = m.group(0) + out, pos = [], 0 + for a in _VISIBLE_ATTR_RE.finditer(tag): + out.append(_stash(tag[pos:a.start(1)], "HTMLTAG")) + out.append(a.group(1)) + pos = a.end(1) + out.append(_stash(tag[pos:], "HTMLTAG")) + return "".join(out) + text = _HTMLTAG_RE.sub(_htmltag, text) + + # 5. Link destinations LAST, so URLs already inside code/shortcodes/HTML tags + # are covered by their own placeholders and are not double-masked. Link TEXT + # stays exposed and still gets translated; only the target is opaque. + text = _AUTOLINK_RE.sub(_sub("URL"), text) + text = _LINKDEST_RE.sub( + lambda m: "(" + _stash(m.group("dest"), "URL") + (m.group("title") or "") + ")", text) + text = _REFDEF_RE.sub( + lambda m: m.group("label") + _stash(m.group("dest"), "URL"), text) + return text, store + + +def restore(text: str, store: dict[str, str]) -> str: + """Substitute placeholders back, repeatedly, until none are left. + + A single pass is not enough because protected spans nest: a shortcode + written inside inline code (`` `{{< version-pin "x" >}}` ``) is stashed as + SC first, and the inline-code pass then stashes the text that already + contains that token. Restoring in insertion order reveals the SC token only + after its own turn has passed, so the published page kept a literal + `§§SC_23§§`. Iterating to a fixed point handles any nesting depth; the loop + is bounded by the number of placeholders, and stops early once a pass + changes nothing. + """ + for _ in range(len(store) + 1): + before = text + for token, original in store.items(): + if token in text: + text = text.replace(token, original) + if text == before: + break + return text + + +# Hugo `ref`/`relref` shortcodes. Both argument shapes Hugo accepts: +# positional: {{% ref "/docs/v1.5/getting-started/install-talos" %}} +# {{< relref "install-cozystack#section" >}} +# named: {{% ref path="/docs/v1.5/x" lang="de" %}} +# The whole call is captured (args blob between `ref` and the closing delimiter), +# then `_ref_target` pulls the target out of either shape. +_REF_SHORTCODE = re.compile(r'\{\{[%<]\s*(?:rel)?ref\s+([^%>]*?)\s*[%>]\}\}') +_REF_PATH_NAMED = re.compile(r'\bpath\s*=\s*"([^"]+)"') +# Any residual ref/relref shortcode at all — used for the fail-closed check. +_ANY_REF_SHORTCODE = re.compile(r'\{\{[%<]\s*(?:rel)?ref\b') + + +def _ref_target(args: str) -> str | None: + """Extract the link target from a ref/relref args blob, or None if it cannot + be determined (an unsupported shape — the caller then fails closed).""" + named = _REF_PATH_NAMED.search(args) + if named: + return named.group(1) + args = args.strip() + if not args: + return None + first = args.split()[0].strip('"').strip("'").strip("`") + # If a quote/backtick/`=`/space survived stripping, this is a shape we do not + # understand (unbalanced quoting, an unhandled named param): return None so + # has_ref_shortcode fires and translate_page fails closed rather than ship a + # silently-broken link. + if not first or any(c in first for c in '"\'`= '): + return None + return first + + +def has_ref_shortcode(text: str) -> bool: + """True if any ref/relref shortcode remains — a build-breaking hazard.""" + return bool(_ANY_REF_SHORTCODE.search(text)) + + +# ---- deterministic integrity & typography checks ---------------------------- +# +# The masking in protect() guarantees that code, shortcodes and URLs survive +# byte-for-byte. What it CANNOT guarantee is the material that legitimately sits +# in ordinary prose: a bare `--flag`, a `kind: Tenant` written without backticks, +# a version number, a brand from do_not_translate. Those are defended only by +# prompt rules, which is to say softly. These checks turn "the model was told +# not to" into "we looked". They return FINDINGS (fed to the same revise loop as +# the reviewers) rather than raising: a false positive should cost one revise +# round, never a hard-failed page. + +_CODEISH_RE = re.compile( + r"```.*?```" # fenced code + r"|`[^`\n]+`" # inline code + r"|§§[A-Z]+_\d+§§" # pipeline placeholders + r"|\{\{[<%].*?[%>]\}\}" # Hugo shortcodes (params are not prose) + r"|<!--.*?-->" # HTML comments + r"|<[^>\n]+>" # HTML tags with their attributes + # Link/image destinations, with the optional "title". The title's ASCII + # quotes are CommonMark delimiters — flagging them would send the revise + # loop to «localize» them into a form that no longer parses as a link. + r'|\]\([^)\s]*(?:\s+"[^"]*")?\)' + r"|!\[" # image marker — its `!` is syntax, not prose + r"|^\[[^\]]+\]:[ \t]*\S+", # reference-link definitions + re.DOTALL | re.MULTILINE) + + +def _prose_only(text: str) -> str: + """Drop everything that is markup or code, so the checks see prose alone. + + Without this the typography rules fire on things that are *correctly* written + with ASCII punctuation — an HTML attribute (`class="hero"`), a shortcode + param, a file path — and a check that cries wolf is a check reviewers learn + to ignore. + """ + return _HTML_ENTITY_RE.sub(" ", _CODEISH_RE.sub(" ", text)) + + +# `→`, `→`, `→` are markup: the trailing `;` is not punctuation, +# and the zh-cn rules would otherwise read it as a half-width semicolon. +_HTML_ENTITY_RE = re.compile(r"&(?:[A-Za-z][A-Za-z0-9]+|#\d+|#x[0-9A-Fa-f]+);") + + +# Version tokens: v1.5, v1.2.5, 1.2.3. Localizing the separator (1,2,3) or +# bumping a digit changes documented behaviour, so counts must match the source. +# Only UNAMBIGUOUS shapes are enforced — a `v` prefix or three components. A +# bare `2.5` in prose is usually a quantity, and the style guides mandate the +# decimal comma for those ("0.5 vCPU" → "0,5 vCPU"); enforcing byte-for-byte +# survival there would flag every correctly localized number, refire the same +# finding every revise round, and permanently block the gate. Bare two-part +# decimals are the reviewers' job. +_VERSION_RE = re.compile(r"\bv\d+\.\d+(?:\.\d+)?\b|\b\d+\.\d+\.\d+\b") +# Correctly LOCALIZED numbers that the bare three-component branch would +# misread as invented versions: numeric dates (24.07.2026 — a component has +# 4 digits) and period-grouped thousands (10.000.000). Both are formats the +# style guides mandate for translated prose, so their appearance in the +# translation alone is expected, not an invention. +_LOCALIZED_NUMBER_RE = re.compile(r"\d+\.\d+\.\d{4}|\d{1,3}(?:\.\d{3}){2,}") +# Bare long CLI flags in prose. +_FLAG_RE = re.compile(r"(?<![\w-])--[a-z][a-z0-9-]{2,}\b") + + +def integrity_findings(src_body: str, tr_body: str, dnt_terms: list[str] | None = None, + who: str = "integrity-check") -> list[dict]: + """Compare source and translation for material that must survive verbatim. + + Checks, all on prose only (masked spans are already guaranteed): + * version tokens — same multiset + * bare CLI flags — same multiset + * do_not_translate — a term present in the source must not vanish + """ + out: list[dict] = [] + src, tr = _prose_only(src_body), _prose_only(tr_body) + + def _counts(rx, s): + d: dict[str, int] = {} + for m in rx.finditer(s): + d[m.group(0)] = d.get(m.group(0), 0) + 1 + return d + + for label, rx in (("version", _VERSION_RE), ("CLI flag", _FLAG_RE)): + a, b = _counts(rx, src), _counts(rx, tr) + for tok, n in a.items(): + got = b.get(tok, 0) + if got < n: + out.append({"severity": "major", "from": who, + "issue": f"{label} '{tok}' appears {n}× in the English source but " + f"{got}× in the translation — it must be carried over " + f"unchanged"}) + for tok in b: + if tok not in a and not _LOCALIZED_NUMBER_RE.fullmatch(tok): + out.append({"severity": "minor", "from": who, + "issue": f"{label} '{tok}' appears in the translation but not in " + f"the source — do not invent versions or flags"}) + + for term in (dnt_terms or []): + # Word-boundary matching, not substring: `Go` must not count inside + # `Google`. `(?!\w)` still allows hyphenated compounds ("Cozystack- + # Plattform" is how German writes it) and possessives. + term_rx = re.compile(rf"(?<!\w){re.escape(term)}(?!\w)") + n = len(term_rx.findall(src)) + got = len(term_rx.findall(tr)) + if n and got < n: + out.append({"severity": "major", "from": who, + "issue": f"do-not-translate term '{term}' appears {n}× in the source " + f"but only {got}× in the translation — it must be " + f"kept verbatim, not translated or transliterated"}) + return out + + +# Per-language typography. The style guides state these rules; nothing checked +# them, so a page could read fine and still be typeset like English. Findings are +# advisory-but-real: they are the cheapest quality signal we have per language. +_TYPO_RULES: dict[str, list[tuple[str, str]]] = { + "ru": [ + (r'"[^"\n]{1,80}"', 'ASCII double quotes in Russian prose — use «ёлочки»'), + # The English pair is “…” (U+201C…U+201D). U+201C alone must not count: + # it CLOSES the nested „лапки“ the guide mandates, so a line with two + # nested pairs («…„replicas“ и „selector“…») would otherwise match. + (r'“[^\n]{0,80}”', 'English curly quotes in Russian prose — use «ёлочки»'), + # A hyphen doing a dash's job — but NOT a list marker ("\n- item"), which + # is why a non-space character is required immediately before it. + (r'(?<=[^\s])[ ]-[ ](?=\S)', 'hyphen used as a dash — use the em dash «—»'), + ], + "de": [ + (r'"[^"\n]{1,80}"', 'ASCII double quotes in German prose — use „…“'), + (r'[“][^\n]{0,80}[”]', 'English curly quotes in German prose — use „…“'), + ], + "es": [ + # Anchored to sentence START (line start or after end punctuation), not + # to any capitalized word — otherwise every brand name inside a + # correctly opened «¿…?» re-matches from the brand and flags correct + # text. The span also stops at a mid-sentence ¿/¡: the guide mandates + # the mark where the question actually starts («Si el nodo falla, + # ¿qué pasa?» is the ✓ form and must not be flagged). + (r'(?m)(?:^|(?<=[.!?:] ))[A-ZÁÉÍÓÚÑ][^.!?¿\n]{5,120}\?', + 'question without an opening ¿'), + (r'(?m)(?:^|(?<=[.!?:] ))[A-ZÁÉÍÓÚÑ][^.!?¡\n]{5,120}!', + 'exclamation without an opening ¡'), + ], + "pt-br": [ + (r'\b(ficheiro|utilizador|ecrã|rato|autocarro|telemóvel|casa de banho)\b', + 'European Portuguese vocabulary leaked into pt-BR'), + (r'\bestá a [a-zçãéêó]+r\b', 'European Portuguese "está a fazer" — pt-BR uses the gerund'), + ], + "zh-cn": [ + (r'[一-鿿]\s*[,;:!?]', 'half-width punctuation after a Chinese character — use ，；：！？'), + (r'[,;:!?]\s*[一-鿿]', 'half-width punctuation before a Chinese character — use ，；：！？'), + (r'[一-鿿]\.(?=\s|$)', 'half-width period ending a Chinese sentence — use 。'), + (r'[Ａ-Ｚａ-ｚ０-９]', 'full-width Latin letters or digits — use half-width'), + ], + "hi": [ + (r'[०-९]', 'Devanagari digits — use Western Arabic numerals in technical content'), + ], +} + + +def check_typography(text: str, lang: str, who: str = "typography-check") -> list[dict]: + """Per-language typography findings (prose only; code spans are exempt).""" + prose = _prose_only(text) + out: list[dict] = [] + for pattern, message in _TYPO_RULES.get(lang, []): + hits = re.findall(pattern, prose) + if hits: + sample = hits[0] if isinstance(hits[0], str) else str(hits[0]) + out.append({"severity": "minor", "from": who, + "issue": f"{message} ({len(hits)}×, e.g. {sample.strip()[:60]!r})"}) + return out + + +def deref_shortcodes(text: str, page_rel: str) -> str: + """Rewrite Hugo ref/relref shortcodes to plain absolute links. + + `ref`/`relref` are resolved at build time WITHIN the current language and + hard-fail (REF_NOT_FOUND) when the target page has not been translated into + that language yet — the normal state of a partially-translated site, which + would break the entire Hugo build. Plain links do not: the render-link hook + (`layouts/_default/_markup/render-link.html`) resolves them to the in-language + page when it exists and otherwise falls back to the unprefixed English page. + This mirrors the convention established in commit 5ca8e55. + + A shortcode whose target cannot be extracted is left untouched on purpose, so + `has_ref_shortcode` can catch it and the caller can fail closed rather than + ship a page that may break the build.""" + def _sub(m: "re.Match[str]") -> str: + target = _ref_target(m.group(1)) + if target is None: + return m.group(0) + path, _, frag = target.partition("#") + if not path: + # Anchor-only ref (`#section`) means "the current page". + path = "/" + page_rel + elif not path.startswith("/"): + # Relative target: resolve against the current page's directory. + path = os.path.normpath(os.path.join("/" + os.path.dirname(page_rel), path)) + path = path.rstrip("/") + if path.endswith(".md"): + path = path[:-3] + # A section index (`.../_index` or `.../index`) is served at the parent + # URL, not at `.../_index/`. + for suffix in ("/_index", "/index"): + if path.endswith(suffix): + path = path[: -len(suffix)] + break + return path + "/" + (f"#{frag}" if frag else "") + return _REF_SHORTCODE.sub(_sub, text) + + diff --git a/hack/i18n/lint_translations.py b/hack/i18n/lint_translations.py new file mode 100755 index 000000000..c906ee0cb --- /dev/null +++ b/hack/i18n/lint_translations.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Lint already-published translations for per-language typography. + +WHY: `check-i18n.sh` guards i18n key parity and translation freshness, and the +pipeline's gate checks a page at the moment it is generated. Neither looks at a +page again afterwards — so a hand edit (or a page translated before a style rule +existed) can sit in the tree indefinitely with English quotation marks in Russian +prose or half-width punctuation in Chinese. The style guides state these rules; +this makes them enforceable on every PR. + +Findings are advisory by default (exit 0) because typography rules are heuristic +and the tree has history. Pass --strict to fail the build instead, once a +language's backlog is clean. + +Usage: + python3 hack/i18n/lint_translations.py # all configured languages + python3 hack/i18n/lint_translations.py --lang ru # one language + python3 hack/i18n/lint_translations.py --strict # exit 1 on any finding +""" +from __future__ import annotations + +import argparse +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import lib + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--lang", help="only this language code") + ap.add_argument("--strict", action="store_true", + help="exit non-zero when findings exist (default: advisory)") + args = ap.parse_args() + + cfg = lib.load_config() + langs = [l["code"] for l in cfg["languages"] if args.lang in (None, l["code"])] + if args.lang and not langs: + print(f"::error::unknown --lang '{args.lang}'", file=sys.stderr) + return 2 + + total = 0 + for lang in langs: + root = os.path.join(lib.REPO_ROOT, cfg["content_dir"], lang) + if not os.path.isdir(root): + continue + for dirpath, _dirs, files in os.walk(root): + for name in sorted(files): + if not name.endswith((".md", ".html")): + continue + path = os.path.join(dirpath, name) + with open(path, encoding="utf-8") as fh: + _fm, body, _raw = lib.split_frontmatter(fh.read()) + findings = lib.check_typography(body, lang) + if findings: + rel = os.path.relpath(path, lib.REPO_ROOT) + print(f"{rel}") + for f in findings: + print(f" - {f['issue']}") + total += len(findings) + + if total: + print(f"\n{total} typography finding(s). These are the rules the per-language " + f"style guides state; fix them or refine the rule in lib._TYPO_RULES.") + else: + print("no typography findings") + return 1 if (total and args.strict) else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/hack/i18n/prompts/back-translate-compare.md b/hack/i18n/prompts/back-translate-compare.md new file mode 100644 index 000000000..73cb75de2 --- /dev/null +++ b/hack/i18n/prompts/back-translate-compare.md @@ -0,0 +1,24 @@ +You are checking a translation for meaning drift using a round-trip test. You +are given the ORIGINAL English page and a BACK-TRANSLATION into English of the +{{LANGUAGE}} translation. Compare them for **semantic fidelity** — did meaning +survive the round trip? + +Report drift where the back-translation shows the {{LANGUAGE}} translation: +- dropped a claim, instruction, sentence, or list item present in the original, +- added a claim not in the original, +- changed a number, version, name, flag, or technical fact, +- reversed or weakened a statement (e.g. "must" → "may", "not supported" → "supported"). + +Ignore differences that are purely stylistic or wording-level and don't change +meaning (back-translation is intentionally literal and will read awkwardly). +Ignore code, shortcodes, URLs, and §§NAME_N§§ placeholders. + +Return strict JSON only: +{ + "verdict": "pass" | "revise", + "findings": [ + {"severity": "major"|"minor", "issue": "<what drifted>", "original": "<original span>", "back": "<back-translated span>"} + ] +} +"pass" = no meaningful drift. "revise" = at least one meaning change. Output ONLY +the JSON object. diff --git a/hack/i18n/prompts/back-translate.md b/hack/i18n/prompts/back-translate.md new file mode 100644 index 000000000..f78dbb8fe --- /dev/null +++ b/hack/i18n/prompts/back-translate.md @@ -0,0 +1,13 @@ +You are a literal back-translator. You are given a page that was translated +from English into {{LANGUAGE}}. Translate it BACK into English as literally and +faithfully as possible — do NOT improve, polish, or correct it. Your job is to +expose what the {{LANGUAGE}} text actually says so it can be compared against the +original English. + +Rules: +- Translate meaning faithfully, including any errors, additions, or omissions in + the {{LANGUAGE}} text. If the translation dropped a sentence, your back- + translation must also lack it. If it added a claim, include that claim. +- Keep §§NAME_N§§ placeholders, code, shortcodes, URLs, and identifiers verbatim. +- Do not consult the original English; translate only what is in front of you. +- Output only the back-translated English text. No commentary, no markers. diff --git a/hack/i18n/prompts/review-editor.md b/hack/i18n/prompts/review-editor.md new file mode 100644 index 000000000..a6da6c8e3 --- /dev/null +++ b/hack/i18n/prompts/review-editor.md @@ -0,0 +1,26 @@ +You are a native {{LANGUAGE}} technical editor and translator reviewing a +machine translation of a page from the Cozystack website (a CNCF cloud-native +platform). You are a NATIVE speaker of {{LANGUAGE}}. Your remit is **literary +quality** — fluency, naturalness, register, terminology consistency, grammar, +and readability for a {{LANGUAGE}}-speaking technical audience. You are NOT +responsible for verifying technical facts (a separate maintainer reviewer does +that) — but flag terminology that reads unnaturally or breaks glossary +consistency. + +Judge against: +- Does it read like a native {{LANGUAGE}} engineer wrote it, not a translation? +- Correct register/tone for the page type (docs = precise; landing/blog = it may + transcreate and read as marketing). +- Grammar, agreement, punctuation, and {{LANGUAGE}} typographic conventions. +- Consistent, natural rendering of recurring terms; preferred terms respected. +- No untranslated leftovers, no awkward calques, no mistranslated idioms. + +Do NOT rewrite the page. Return findings only, as strict JSON: +{ + "verdict": "pass" | "revise", + "findings": [ + {"severity": "major"|"minor", "quote": "<short problematic span>", "issue": "<what's wrong>", "suggestion": "<how to fix>"} + ] +} +"pass" = publishable as-is (minor nits allowed). "revise" = at least one issue +that should be fixed before publish. Output ONLY the JSON object. diff --git a/hack/i18n/prompts/review-maintainer.md b/hack/i18n/prompts/review-maintainer.md new file mode 100644 index 000000000..d3a847b2f --- /dev/null +++ b/hack/i18n/prompts/review-maintainer.md @@ -0,0 +1,28 @@ +You are a Cozystack maintainer and a NATIVE {{LANGUAGE}} speaker, reviewing a +{{LANGUAGE}} translation of a page from the Cozystack website (Kubernetes, +KubeVirt, Cilium, LINSTOR, Talos, storage, networking, virtualization). Your +remit is **technical correctness** — that the translation preserves the exact +technical meaning of the English source. Literary quality is another reviewer's +job; focus on whether an engineer following the {{LANGUAGE}} page would do the +right thing. + +You are given the ENGLISH SOURCE and the {{LANGUAGE}} TRANSLATION. Check that: +- Every technical claim, instruction, and step matches the source — nothing + added, dropped, weakened, or reversed. +- Numbers, versions, flags, resource names, commands, YAML keys, and API/CRD + names are unchanged and correct. +- Brand/product/technology names (Kubernetes, Cozystack, KubeVirt, …) are kept + in Latin script, not translated or transliterated. +- No term was translated in a way that changes its technical meaning (e.g. a + {{LANGUAGE}} word that means something different from the k8s concept). +- Code blocks, shortcodes, and URLs are intact and unaltered. + +Do NOT rewrite the page. Return findings only, as strict JSON: +{ + "verdict": "pass" | "revise", + "findings": [ + {"severity": "major"|"minor", "quote": "<short span in the translation>", "issue": "<technical inaccuracy vs source>", "suggestion": "<correct rendering>"} + ] +} +"pass" = technically faithful (minor nits allowed). "revise" = at least one +technical inaccuracy that must be fixed. Output ONLY the JSON object. diff --git a/hack/i18n/prompts/revise.md b/hack/i18n/prompts/revise.md new file mode 100644 index 000000000..152116f15 --- /dev/null +++ b/hack/i18n/prompts/revise.md @@ -0,0 +1,24 @@ +You are revising a {{LANGUAGE}} translation of a Cozystack website page to +address specific reviewer findings. You are a native {{LANGUAGE}} technical +translator. + +You are given: the ENGLISH SOURCE, the current {{LANGUAGE}} TRANSLATION, and a +list of FINDINGS from a technical editor and a Cozystack maintainer. Produce a +corrected {{LANGUAGE}} translation that resolves every finding while keeping +everything the reviewers did NOT flag unchanged. + +Hard rules (unchanged from the original translation task): +- Preserve §§NAME_N§§ placeholders, code blocks, shortcodes, URLs, CLI, YAML + keys, numbers, versions, and Latin-script brand names verbatim. +- Preserve Markdown/HTML structure exactly (headings, lists, tables, links — + translate link text, keep URLs). +- Do not introduce new content beyond fixing the findings. + +Same output protocol as translation. The user message contains a FRONTMATTER +section (a JSON object) and a BODY section separated by `===FRONTMATTER===` and +`===BODY===`. Respond with the SAME two markers and nothing else: + +===FRONTMATTER=== +<a JSON object with the SAME keys and corrected values> +===BODY=== +<corrected body> diff --git a/hack/i18n/prompts/translate.md b/hack/i18n/prompts/translate.md new file mode 100644 index 000000000..d6f109391 --- /dev/null +++ b/hack/i18n/prompts/translate.md @@ -0,0 +1,52 @@ +You are a professional technical translator localizing the Cozystack website +(a CNCF cloud-native platform: Kubernetes, virtualization, storage, networking) +from English into {{LANGUAGE}} ({{LANG_CODE}}). + +Translate for a technical audience — platform engineers, SREs, CTOs. Produce +natural, idiomatic {{LANGUAGE}} that a native engineer would write, not a literal +word-for-word rendering. Preserve the author's meaning, technical precision, and +tone. When a sentence is marketing copy (landing/blog), transcreate it so it +reads well in {{LANGUAGE}}; when it is documentation, stay precise and faithful. + +## Hard rules + +1. NEVER translate, transliterate, or inflect these terms — keep them exactly as + written, in Latin script: {{DO_NOT_TRANSLATE}} + This list is CASE-SENSITIVE and only matches the exact capitalization shown. + A capitalized term is an API kind or product name (`Tenant` = `kind: Tenant` + in a manifest, which the reader must be able to match); the same word in + lower case is an ordinary noun and IS translated — see the preferred list. +2. NEVER alter placeholders of the form §§NAME_N§§ — copy them through verbatim, + in the same positions. They stand for code blocks, shortcodes, HTML comments, + and inline code that must not change. +3. NEVER change URLs, file paths, CLI commands, flags, API field names, or YAML + keys. Translate surrounding prose only. +4. Preserve all Markdown/HTML structure exactly: heading levels, list markers, + tables (same number of columns and delimiter cells), links (translate link + TEXT, keep the URL), bold/italic, and blank lines. +5. Never change the VALUE of a number, version, or measurement — a figure in the translation must mean exactly what it meant in English. Version strings, numbers inside code, YAML values, CIDRs, Kubernetes quantities (`512Mi`, `500m`), ports, and tags are literal identifiers: reproduce them character-for-character, never localizing their separators. FORMATTING of numbers, dates and units in ordinary PROSE follows the style guide below — several languages require a decimal comma, a different thousands separator, or a different date order. Reformatting prose is expected; changing a value, or touching a version/identifier, is not. + +## Preferred terminology (use consistently) + +{{PREFERRED_TERMS}} + +## Style guide for {{LANGUAGE}} + +{{STYLE_GUIDE}} + +## Output protocol + +The user message contains a FRONTMATTER section (a JSON object mapping an opaque +path to the English text) and a BODY section, separated by the exact markers +`===FRONTMATTER===` and `===BODY===`. Respond with the SAME two markers and +nothing else: + +===FRONTMATTER=== +<a JSON object with the SAME keys and translated values — do not add, drop, or +rename keys, and do not interpret the keys themselves> +===BODY=== +<the translated body> + +For SEO fields (title, description): make them read naturally in {{LANGUAGE}} and +incorporate any provided keyword targets, but never keyword-stuff. Do not add, +remove, or reorder front-matter keys. Do not wrap your answer in code fences. diff --git a/hack/i18n/requirements.txt b/hack/i18n/requirements.txt new file mode 100644 index 000000000..7386d10b1 --- /dev/null +++ b/hack/i18n/requirements.txt @@ -0,0 +1,6 @@ +# Pinned on purpose: run-daily.sh installs into its venv on every invocation, +# on a machine holding maintainer credentials (gh token, Claude OAuth token). +# An unpinned install would silently upgrade code running next to those +# credentials. Bump deliberately, in a reviewed commit. +claude-agent-sdk==0.2.123 +pyyaml==6.0.3 diff --git a/hack/i18n/run-daily.sh b/hack/i18n/run-daily.sh new file mode 100755 index 000000000..96a4d1518 --- /dev/null +++ b/hack/i18n/run-daily.sh @@ -0,0 +1,326 @@ +#!/usr/bin/env bash +# +# run-daily.sh — translation run. Cadence: run this DAILY (cron/launchd). +# +# Daily run, weekly PR: each run uses that day's model quota, and the week's +# output accumulates on a single `i18n/week-<ISO week>` branch. One PR per week +# is opened and updated, and a MAINTAINER merges it — nothing reaches the +# production site automatically, and CODEOWNERS/branch protection are untouched. +# +# RUN THIS IN A DEDICATED CLONE, not your working checkout: it switches branches +# and commits. It refuses to start if the working tree is dirty, and always +# restores the branch you started on. +# +# Auth: see `auth` in hack/i18n/config.yaml. +# oauth-subscription (default): a logged-in `claude` CLI on this machine, or +# CLAUDE_CODE_OAUTH_TOKEN from `claude setup-token`. ANTHROPIC_API_KEY must +# NOT be set — it would shadow the subscription and bill metered API. +# api-key: ANTHROPIC_API_KEY must be set (prefer an org-owned key). +# +# Usage: hack/i18n/run-daily.sh [--lang ru] [--limit N] +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$REPO_ROOT" + +# -------------------------------------------------------------------- run lock +# One run per clone at a time: a manual invocation overlapping a slow cron run +# passes the clean-tree preflight and then races the cron run on checkout, +# commit and push — silently, until the second push fails. A pidfile makes the +# second invocation fail fast and loudly instead. Pidfile rather than flock(1): +# the runner may be a macOS/launchd machine, where flock does not exist. +# Keyed by clone path so two DIFFERENT clones on one machine don't block +# each other. Cleaned by the EXIT trap; a lock whose pid is dead is stale +# (crash, power loss) and is taken over rather than wedging every later run. +LOCK_FILE="${TMPDIR:-/tmp}/cozystack-i18n-$(printf '%s' "$REPO_ROOT" | cksum | cut -d' ' -f1).pid" +if ! (set -o noclobber; echo "$$" > "$LOCK_FILE") 2>/dev/null; then + OTHER_PID="$(cat "$LOCK_FILE" 2>/dev/null || true)" + if [ -n "$OTHER_PID" ] && kill -0 "$OTHER_PID" 2>/dev/null; then + echo "error: another run-daily.sh (pid $OTHER_PID) is already running against" >&2 + echo " this clone — refusing to race it. (lock: $LOCK_FILE)" >&2 + exit 1 + fi + echo "warning: stale lock from pid ${OTHER_PID:-?} (not running) — taking over." >&2 + echo "$$" > "$LOCK_FILE" +fi +trap 'rm -f "$LOCK_FILE"' EXIT + +# ------------------------------------------------------------ durable run log +# A run that stops early on the usage limit or skips every page writes a report, +# but the only GitHub-facing channel for it is a PR comment — and on the first +# run of a week no PR exists yet. Keep a durable, dated copy OUTSIDE the repo +# tree (inside it would trip the next run's clean-tree preflight) so the record +# never depends on the operator having wired up cron log capture. +STATE_DIR="${I18N_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/cozystack-i18n}" +persist_report() { + [ -f "$1" ] || return 0 + local dest + dest="$STATE_DIR/run-$(date -u +%Y-%m-%dT%H%M%SZ).md" + if mkdir -p "$STATE_DIR" 2>/dev/null && cp "$1" "$dest" 2>/dev/null; then + echo "run report saved to $dest" + else + # This is the log-capture-independent record; if it fails, say so loudly + # rather than reproducing the silent-failure class it exists to close. + echo "warning: could not persist run report to $STATE_DIR — see stdout above" >&2 + fi +} + +# ---------------------------------------------------------------- environment +VENV="${I18N_VENV:-$REPO_ROOT/.venv-i18n}" +if [ ! -x "$VENV/bin/python" ]; then + echo "bootstrapping venv at $VENV ..." + python3 -m venv "$VENV" + "$VENV/bin/pip" install --quiet --upgrade pip +fi +# Distro Pythons are PEP 668 "externally managed", so the pipeline runs from a venv. +# Versions are PINNED (requirements.txt): this install runs on every invocation on +# a machine that holds maintainer credentials — it must not upgrade in place. +"$VENV/bin/pip" install --quiet -r "$REPO_ROOT/hack/i18n/requirements.txt" +PY="$VENV/bin/python" + +cfg() { "$PY" - "$1" <<'PY' +import sys; sys.path.insert(0, "hack/i18n"); import lib +cfg = lib.load_config() +key = sys.argv[1] +if key == "langs": + print("\n".join(l["code"] for l in cfg["languages"])) +else: + print(cfg.get(key, "")) +PY +} + +# I18N_AUTH overrides the config's auth mode without editing the tracked file — +# the clean-tree preflight below forbids an in-place edit, so CI (which runs on an +# org api-key while the committed default stays oauth-subscription for local +# bootstrap) sets it in the environment instead. translate.py honours the same var. +AUTH_MODE="${I18N_AUTH:-$(cfg auth)}" +if [ "$AUTH_MODE" = "oauth-subscription" ]; then + if [ -n "${ANTHROPIC_API_KEY:-}" ]; then + echo "error: auth=oauth-subscription but ANTHROPIC_API_KEY is set — it shadows the" >&2 + echo " subscription and would bill metered API. Unset it, or set auth: api-key." >&2 + exit 1 + fi + if [ -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ] && ! command -v claude >/dev/null 2>&1; then + echo "error: no subscription credential — log in with the 'claude' CLI, or run" >&2 + echo " 'claude setup-token' and export CLAUDE_CODE_OAUTH_TOKEN." >&2 + exit 1 + fi +elif [ "$AUTH_MODE" = "api-key" ]; then + if [ -z "${ANTHROPIC_API_KEY:-}" ]; then + echo "error: auth=api-key but ANTHROPIC_API_KEY is not set." >&2 + exit 1 + fi +else + echo "error: unknown auth mode '$AUTH_MODE' in config.yaml" >&2 + exit 1 +fi + +# Publishing needs gh. Without this check an unauthenticated runner would spend a +# full run of model calls and only fail at `gh pr create`. +if ! gh auth status >/dev/null 2>&1; then + echo "error: gh is not authenticated — run 'gh auth login' or set GH_TOKEN." >&2 + exit 1 +fi + +# Publish mode: only pr_only is implemented (push a weekly branch, open a PR for a +# maintainer to merge). Fail closed on anything else rather than fall through to +# some unintended behaviour — there is no auto-merge path in this script. +PUBLISH_MODE="$(cfg publish_mode)" +if [ "$PUBLISH_MODE" != "pr_only" ]; then + echo "error: publish_mode='$PUBLISH_MODE' in config.yaml is not supported (only 'pr_only')." >&2 + exit 1 +fi + +# --------------------------------------------------------------- git preflight +# A dirty tree means we cannot safely switch branches; and stashing the user's +# work has proven to be a footgun (conflicts write markers into translations, +# and an empty stash pops someone else's). +if [ -n "$(git status --porcelain)" ]; then + echo "error: working tree is not clean. Run this in a dedicated clone." >&2 + echo " If a previous run crashed mid-run, the leftovers are pages that" >&2 + echo " finished their gate but were never committed. Cheapest recovery:" >&2 + echo " discard them and let the next run redo those pages:" >&2 + echo " git checkout -- content/ && git clean -fd content/" >&2 + git status --short >&2 + exit 1 +fi +START_REF="$(git symbolic-ref --quiet --short HEAD || git rev-parse HEAD)" +restore_branch() { git checkout --quiet "$START_REF" 2>/dev/null || true; } +# Single EXIT trap: bash keeps only the last one, so fold the lock cleanup in +# rather than losing it here. +trap 'restore_branch; rm -f "$LOCK_FILE"' EXIT + +# ------------------------------------------------- sync to the publish branch +# The week branch is checked out BEFORE translating, not after. Translating on +# whatever branch the clone happened to be on and switching only to publish +# wedged the runner on its second run: the EXIT trap restored the start branch, +# which removed day 1's translations from the working tree, so day 2 re-translated +# them (the worklist reads the filesystem) and then collided with the very same +# files as untracked on checkout. Working on the branch the commits belong to +# keeps the worklist honest, kills the collision, and refreshes content/en in +# the same move. +# +# One branch per ISO week. --prune matters: without it a remote-tracking ref for +# a merged+deleted branch lingers, and we would resurrect it. +git fetch --quiet --prune origin +BRANCH="i18n/week-$(date -u +%G-W%V)" + +# First, re-push any week branch a failed push left stranded locally. Waiting +# for new work to trigger the next push is not enough: with an empty backlog +# the run exits before publishing, and at a week boundary the new branch bases +# on origin refs — either way a locally-committed day would sit unpublished +# (and, across weeks, be re-translated). Push failure here is a warning, not a +# death: the commits stay local and the next run retries. +for LOCAL in $(git for-each-ref --format='%(refname:short)' \ + 'refs/heads/i18n/week-[0-9][0-9][0-9][0-9]-W[0-9][0-9]'); do + if ! git rev-parse --verify --quiet "origin/$LOCAL" >/dev/null; then + # Missing on origin means EITHER the first push failed (genuinely stranded) + # OR the branch was merged/closed upstream and deleted, while our local copy + # survived. Only the former may be pushed: re-pushing a merged branch + # resurrects it on origin every morning, and the new-week base selection + # below would then stack every later week on the zombie — freezing this + # clone's content/en at that week's snapshot forever. PR state is the + # discriminator (`gh pr view` resolves a branch's most recent PR regardless + # of state; a merge-base ancestor check would break under squash merges). + STATE="$(gh pr view "$LOCAL" --json state -q .state 2>/dev/null || echo NONE)" + if [ "$STATE" = "MERGED" ] || [ "$STATE" = "CLOSED" ]; then + # The work is on origin/main (merged) or was rejected (closed) — either + # way the local copy is a zombie. 2>/dev/null: deleting fails harmlessly + # if this branch happens to be checked out; the next run retries. + git branch --quiet -D "$LOCAL" 2>/dev/null || true + continue + fi + git push --quiet -u origin "refs/heads/$LOCAL:refs/heads/$LOCAL" \ + || echo "warning: could not push stranded $LOCAL — its commits stay local, will retry" >&2 + elif [ -n "$(git rev-list "origin/$LOCAL..refs/heads/$LOCAL" 2>/dev/null)" ]; then + git push --quiet -u origin "refs/heads/$LOCAL:refs/heads/$LOCAL" \ + || echo "warning: could not push stranded $LOCAL — its commits stay local, will retry" >&2 + fi +done + +if git rev-parse --verify --quiet "origin/$BRANCH" >/dev/null; then + # Continue this week's branch — but never discard local commits that a failed + # push left ahead of origin (a GitHub outage on day 1 must not cost day 1). + if git rev-parse --verify --quiet "refs/heads/$BRANCH" >/dev/null && \ + [ -n "$(git rev-list "origin/$BRANCH..refs/heads/$BRANCH" 2>/dev/null)" ]; then + git checkout --quiet "$BRANCH" + else + git checkout --quiet -B "$BRANCH" "origin/$BRANCH" + fi +elif git rev-parse --verify --quiet "refs/heads/$BRANCH" >/dev/null; then + # origin never saw this branch (the very first push failed) — keep the local one. + git checkout --quiet "$BRANCH" +else + # First run of a new week. Base on the NEWEST still-existing week branch, not + # origin/main: if last week's PR is not merged yet (maintainers do skip + # weeks), its translations are not on main — cutting from main would forget + # them, re-translate the lot on a day's quota, and open a second PR that + # conflicts with the first. Stacking on the unmerged branch keeps the work; + # once the older PR merges, this week's PR shrinks to its own commits. + # `fetch --prune` above already dropped refs of merged-and-deleted branches. + # The glob is deliberately strict (YYYY-WNN): a stray alphabetic branch like + # i18n/week-test would sort above every dated branch and become the base. + PREV_WEEK="$(git for-each-ref --format='%(refname:short)' --sort=-refname \ + 'refs/remotes/origin/i18n/week-[0-9][0-9][0-9][0-9]-W[0-9][0-9]' | head -1)" + git checkout --quiet -B "$BRANCH" "${PREV_WEEK:-origin/main}" +fi + +# ------------------------------------------------------------------- translate +echo "== i18n run $(date -u +%Y-%m-%dT%H:%MZ) (auth=$AUTH_MODE) ==" +# `| head` closes the pipe and makes Python raise BrokenPipeError, which under +# `set -o pipefail` would kill the run before a single page is translated. +"$PY" hack/i18n/worklist.py ${1:+"$@"} 2>/dev/null | head -3 || true + +# Translates until the daily quota stops it; exits 0 on a usage limit. +"$PY" hack/i18n/translate.py "$@" + +# --------------------------------------------------------------- collect output +LANG_PATHS=() +while IFS= read -r code; do + # Only existing dirs: `git stash`/`git add` fail hard on a pathspec that + # matches nothing (a language with no content yet), whereas `git status` does not. + [ -d "content/$code" ] && LANG_PATHS+=("content/$code") +done < <(cfg langs) + +if [ ${#LANG_PATHS[@]} -eq 0 ] || [ -z "$(git status --porcelain -- "${LANG_PATHS[@]}")" ]; then + echo "no new translations this run — nothing to publish." + # Nothing to publish still isn't nothing to report: the run may have stopped + # early on the usage limit or skipped pages on error. Persist the report to the + # durable state dir unconditionally, and also post it to this week's PR when one + # is already open. On the first run of a week (no PR yet) the state-dir copy is + # the record — no dependence on the operator having wired up cron log capture. + FINDINGS="hack/i18n/last-run-findings.md" + persist_report "$FINDINGS" + if [ -f "$FINDINGS" ] && \ + [ "$(gh pr view "$BRANCH" --json state -q .state 2>/dev/null || echo NONE)" = "OPEN" ]; then + { printf '#### Run %s (no new pages published)\n\n' "$(date -u +%Y-%m-%d\ %H:%MZ)"; cat "$FINDINGS"; } \ + | gh pr comment "$BRANCH" --body-file - + fi + exit 0 +fi + +# Freshness/parity is informational here: "some pages are still stale" is the +# normal steady state of a quota-limited backlog, so it must not block +# publishing the work this run DID finish. +if ! ./hack/check-i18n.sh; then + echo "::warning::check-i18n.sh reported gaps (expected while the backlog drains)." +fi + +# ------------------------------------------------------------------- publish +# Already on $BRANCH (checked out before translating) — commit in place. +git config user.name "$(git config user.name || echo cozystack-i18n)" +git config user.email "$(git config user.email || echo noreply@cozystack.io)" + +git add -- "${LANG_PATHS[@]}" +if git diff --cached --quiet; then + echo "nothing staged — nothing to publish." + exit 0 +fi +git commit --quiet --signoff -m "i18n: machine-reviewed translation update ($(date -u +%Y-%m-%d)) + +Machine review gate: translation, back-translation meaning check, and two +native virtual reviewers (technical editor + Cozystack maintainer). Pages that +did not clear the gate are stamped auto-reviewed-with-findings. Native owners +ratify asynchronously; source_digest tracks freshness." +git push --quiet -u origin "$BRANCH" + +# Open a PR only if this week's branch has no OPEN one. `gh pr view` resolves the +# branch's most recent PR regardless of state, so a merged PR would otherwise +# look like "a PR exists" and the week's later work would sit on an orphan branch. +PR_BODY="Automated translation run. Touches \`content/<lang>/\` only — no changes to code, layouts, config, or the English source. + +This PR collects **this week's** translations; the pipeline runs daily and pushes onto this branch, so please review and merge it at the end of the week. + +Each page went through the machine review gate: translation, a back-translation meaning check, and two virtual reviewers prompted as native speakers (technical editor + Cozystack maintainer). Pages that cleared it are stamped \`translation_review: auto-reviewed\`; pages where findings stayed open are stamped \`auto-reviewed-with-findings\` — review those first. The reviewers are the same model as the translator, so this gate measures self-consistency, not human ratification. + +Every localized page carries a machine-translation banner linking to the English original until a native speaker sets \`translation_review: ratified\`." + +PR_STATE="$(gh pr view "$BRANCH" --json state -q .state 2>/dev/null || echo NONE)" +if [ "$PR_STATE" != "OPEN" ]; then + # No `|| true` here (or on the comment below): commits are already pushed, so a + # failed `gh` call must fail the run — a cron exit 0 with pushed commits and no + # PR would be indistinguishable from a good run. + gh pr create --base main --head "$BRANCH" \ + --title "i18n: translation update, week $(date -u +%G-W%V)" \ + --body "$PR_BODY" +fi + +# Surface what the reviewers actually found. A "-with-findings" stamp with no +# record of the finding is not something a maintainer can act on. +# +# This goes in a COMMENT, not the PR body: the body would have to be rewritten +# every day, and each rewrite would drop the previous days' findings — the PR +# accumulates a week of runs, but the report file only ever holds the last one. +# Comments accumulate on their own, so the thread ends up being the week's log. +FINDINGS="hack/i18n/last-run-findings.md" +persist_report "$FINDINGS" +if [ -f "$FINDINGS" ]; then + { printf '#### Run %s\n\n' "$(date -u +%Y-%m-%d\ %H:%MZ)"; cat "$FINDINGS"; } \ + | gh pr comment "$BRANCH" --body-file - +fi + +# There is deliberately no auto-merge code path: "nothing reaches the production +# site without a maintainer merging it" is a property of this script, not a +# default that one config word away stops being true. +echo "PR left open for a maintainer to review and merge (weekly)." +echo "== done ==" diff --git a/hack/i18n/style-guides/de.md b/hack/i18n/style-guides/de.md new file mode 100644 index 000000000..f52c7b883 --- /dev/null +++ b/hack/i18n/style-guides/de.md @@ -0,0 +1,101 @@ +Code blocks, inline code, Hugo shortcodes, URLs, front-matter keys and `§§NAME_N§§` placeholders are masked structurally by the pipeline — never translate, reorder or "fix" anything inside them. Everything below applies to prose only. + +## 1. Register and tone + +- Formal register: address the reader with **„Sie“** — docs, blog, landing pages, UI strings, notes and warnings alike. Never „du“, never a register switch inside a page. +- Docs: precise, neutral, instructional. Blog/landing: same „Sie“, but shorter sentences, active voice, and transcreated marketing claims instead of word-for-word renderings. +- Instructions use the Sie-imperative (verb first): "Create a namespace." → ✗ „Du erstellst einen Namespace.“ / ✗ „Erstellen ein Namespace.“ → ✓ „Erstellen Sie einen Namespace.“ +- In numbered step lists the Infinitiv is acceptable („Cluster anlegen“), but never mix Infinitiv and Sie-Imperativ within one list. +- "Please note" → ✓ „Beachten Sie“ (not „Bitte beachten Sie bitte“). "Let's create…" → ✓ „Erstellen Sie …“ / „Als Nächstes …“, never „Lassen Sie uns …“. + +## 2. Headings + +- German capitalization: nouns and the first word up, everything else down. English title case must not survive: "Installing the Cozystack Platform" → ✗ „Installieren Der Cozystack Plattform“ → ✓ „Cozystack-Plattform installieren“. +- Task headings: Infinitivkonstruktion. "Configure networking" → ✓ „Netzwerk konfigurieren“ (not „Konfigurieren Sie das Netzwerk“). Concept headings: nominal — "Storage architecture" → ✓ „Speicherarchitektur“. +- Idiomatic, not literal: "Getting started" → ✓ „Erste Schritte“; "Troubleshooting" → ✓ „Fehlerbehebung“; "Prerequisites" → ✓ „Voraussetzungen“; "What's next" → ✓ „Wie geht es weiter“. +- No trailing period; keep question headings as questions; keep heading levels exactly as in the source. + +## 3. Terminology, API objects, gender, plural + +Decision rule for a term **not** covered by the injected glossary, applied in this order: +1. Is it an identifier — API kind, CRD, field, flag, command, chart or package name? → keep verbatim English, uninflected internally, never translated. +2. Does kubernetes.io/de use an established German rendering? → use it. +3. Does the CNCF Cloud Native Glossary (de) have one? → use it. +4. Would a German platform engineer say the English word out loud (Deployment, Workload, Pod, Container, Ingress, Backup, Image)? → keep the English loanword. +5. Otherwise translate, and gloss once on first use: „Mandantentrennung (multi-tenancy)“. Never render the same term two ways on one page. + +- API objects stay English and capitalized in prose: „ein Pod“, „das Deployment“, „ein Secret“, „die VirtualMachine“. Do not translate an object kind (✗ „Geheimnis“ for `Secret`); use „Knoten“ for a cluster node as a general concept, `Node` when the API object is meant. +- Gender (a wrong-but-consistent article is a minor issue; a fluctuating one is major): **der** Cluster, der Pod, der Node, der Container, der Namespace, der Service, der Controller, der Ingress, der Server, der Speicher. **das** Deployment, das Secret, das Volume, das Image, das Manifest, das Backup — plus every `-ment` and every `-ing` noun (das Logging, das Monitoring, das Networking, das Scheduling). **die** Workload, die Ressource, die Instanz, die Pipeline. +- Acronyms take the gender of the German head noun: die API (Schnittstelle), die CRD (Definition), die CPU (Einheit), der RAM (Speicher), die IP-Adresse, das YAML (Format). +- Plural: English `-s` for most loanwords (die Pods, die Nodes, die Deployments, die Volumes, die Namespaces); **no** `-s` for `-er`/`-el` stems (der Cluster → die Cluster, der Container → die Container, der Server → die Server). Genitive singular takes `-s`: „des Pods“, „des Clusters“, „des Deployments“. +- Separate act from object: "the deployment of the app" → „die Bereitstellung der Anwendung“; the `Deployment` object → „das Deployment“. "to deploy" → „bereitstellen“/„ausrollen“, not „deployen“. + +## 4. Compound nouns (the biggest German MT failure mode) + +- All-German compounds are written as one word: ✗ „Speicher Klasse“ → ✓ „Speicherklasse“; ✗ „Netzwerk Richtlinie“ → ✓ „Netzwerkrichtlinie“; ✗ „Konfigurations Datei“ → ✓ „Konfigurationsdatei“. A space between the parts of a compound (Deppenleerzeichen) is always an error. +- Mixed English-German compounds are hyphenated at every joint: ✓ „Kubernetes-Cluster“, „Pod-Netzwerk“, „Container-Image“, „Cluster-Konfiguration“, „Node-Ausfall“, „Speicher-Backend“. +- Multiword English terms used as one noun get Durchkopplung (all joints hyphenated): ✗ „Control Plane Knoten“ → ✓ „Control-Plane-Knoten“; ✗ „Bare Metal Server“ → ✓ „Bare-Metal-Server“; ✓ „High-Availability-Setup“, „Multi-Tenant-Umgebung“, „Kubernetes-as-a-Service-Angebot“. +- Identifiers keep their exact original spelling inside a compound: „YAML-Datei“, „kubectl-Befehl“, „CSI-Treiber“, „VirtualMachine-Ressource“, „Helm-Chart“. +- Readability beats mechanical joining: with three or more elements, hyphenate the load-bearing joint or reformulate. ✗ „Kubernetesclusterkonfigurationsdatei“ → ✓ „Konfigurationsdatei des Kubernetes-Clusters“. +- Do not hyphenate a plain German compound that should be closed (✗ „Speicher-Klasse“ where „Speicherklasse“ exists), and never lowercase the resulting noun: ✓ „das Cluster-Upgrade“. + +## 5. Typography and punctuation + +- Quotation marks: „…“ (U+201E … U+201C); nested ‚…‘ (U+201A … U+2018). Never straight `"`, never English “…”. Do not put quotes around code — it is masked. +- Gedankenstrich: en dash **–** with spaces around it; never the em dash —. "Cozystack — a platform for…" → ✓ „Cozystack – eine Plattform für …“. Ranges use – without spaces: „3–5 Nodes“. The hyphen `-` is reserved for compounds. +- Ellipsis: the single character „…“, with a preceding space when it stands for omitted words. +- No Oxford comma: "A, B, and C" → ✓ „A, B und C“. +- Commas German requires and English does not: before every subordinate clause (dass, weil, wenn, ob, obwohl, relative clauses) and before extended infinitive groups with um/ohne/statt … zu. "We recommend that you upgrade first." → ✓ „Wir empfehlen, dass Sie zuerst aktualisieren.“; ✓ „…, um den Cluster zu aktualisieren.“ +- Lists: sentence fragments take no final period, full sentences do — consistently within one list. After a colon, continue lowercase unless a full sentence or a noun follows. + +## 6. Numbers, dates, units + +- Decimal **comma** in prose: "3.14" → ✓ „3,14“; "0.5 vCPU" → ✓ „0,5 vCPU“. CRITICAL: never convert inside code, version numbers (v1.5.0), image tags, IP addresses/CIDRs, ports, CLI output or YAML — those keep the source form byte-for-byte. +- Thousands separator: period, or a narrow space: "10,000 pods" → ✓ „10.000 Pods“. Prose only, same exception list as above. +- Dates: "07/24/2026" → ✓ „24.07.2026“ or „24. Juli 2026“. Never keep US month/day order. ISO dates in front matter and code stay ISO. Times: "3 PM" → ✓ „15:00 Uhr“. +- Units: space between number and unit, percent included — „4 GB“, „10 Gbit/s“, „50 %“, „25 °C“ (non-breaking space preferred). Kubernetes quantities (512Mi, 2000m) are code-like and stay unchanged. +- Ordinals take a period: „im 3. Schritt“. "billion" → „Milliarde“ (never „Billion“). + +## 7. Grammar traps when translating from English + +- Verbzweitstellung: the finite verb is the second element of a main clause. ✗ „Mit diesem Befehl Sie erstellen einen Cluster.“ → ✓ „Mit diesem Befehl erstellen Sie einen Cluster.“ +- Verbendstellung in subordinate clauses. ✗ „…, weil der Cluster hat nicht genug Speicher.“ → ✓ „…, weil der Cluster nicht genug Speicher hat.“ +- Modal verbs push the infinitive to the end. ✗ „Sie können erstellen eine virtuelle Maschine.“ → ✓ „Sie können eine virtuelle Maschine erstellen.“ +- Passive: prefer active or „lässt sich“; avoid stacked „wird … werden“ chains. "The cluster can be upgraded without downtime." → ✓ „Der Cluster lässt sich ohne Ausfallzeit aktualisieren.“ For a generic actor use the passive or man-Konstruktion, not „Sie“: "Backups are stored in S3." → ✓ „Backups werden in S3 abgelegt.“ +- Capability, not permission: "Cozystack allows you to run VMs" → ✗ „Cozystack erlaubt Ihnen, VMs auszuführen“ → ✓ „Mit Cozystack können Sie VMs betreiben“ / „Cozystack ermöglicht den Betrieb von VMs“. +- Gerunds are not present participles: "Using kubectl, you can…" → ✗ „Benutzend kubectl …“ → ✓ „Mit kubectl können Sie …“; "After installing…" → ✓ „Nach der Installation …“. +- Genitiv over „von“ in docs: ✓ „die Konfiguration des Clusters“; „von“ only without article or with bare plurals: ✓ „eine Liste von Nodes“. +- Restore articles English drops: "Create Deployment" → ✓ „Erstellen Sie ein Deployment“. Watch case after prepositions: „mit dem Cluster“, „für den Node“, „in der Umgebung“. +- No English noun stacking: ✗ „Cozystack Kubernetes Plattform“ → ✓ „die Kubernetes-Plattform Cozystack“. + +## 8. Translationese and calques to eliminate + +- ✗ „In diesem Artikel werden wir …“ → ✓ „Dieser Artikel zeigt, wie …“ / „Im Folgenden …“. +- ✗ filler „einfach“/„simpel“ ("simply run") → ✓ delete it: „Führen Sie … aus.“ +- ✗ „erlaubt Ihnen zu …“ → ✓ „ermöglicht …“ / „damit können Sie …“. +- ✗ dangling „basierend auf“ at the start of a clause → ✓ „auf Basis von“ / „anhand von“. +- ✗ „in Ordnung sein“ for "to be fine/OK" → ✓ „funktioniert“ / „ist zulässig“; ✗ „Sie wollen vielleicht …“ (you may want to) → ✓ „Optional können Sie …“. +- ✗ „Stellen Sie sicher, dass Sie X getan haben“ → ✓ „Vergewissern Sie sich, dass X …“ / „Voraussetzung: …“. +- ✗ „unterstützt“ for every sense of "support": feature support ✓ „unterstützt“, commercial support ✓ „Support“/„Betreuung“, "supported version" ✓ „unterstützte Version“. +- ✗ „adressieren“ (to address an issue) → ✓ „beheben“/„angehen“; ✗ „realisieren“ (to realize) → ✓ „umsetzen“ or „feststellen“. +- ✗ „Einmal der Cluster läuft, …“ → ✓ „Sobald der Cluster läuft, …“; ✗ over-nominalization „die Durchführung der Installation“ → ✓ „die Installation“; ✗ „Technologien wie z. B. …“ → ✓ „Technologien wie …“. + +## 9. False friends (EN ≠ DE) + +eventually ≠ eventuell → „schließlich/letztendlich“ (eventual consistency → „letztendliche Konsistenz“); actual/actually ≠ aktuell → „tatsächlich“ (current → „aktuell“); to control ≠ kontrollieren → „steuern/regeln“ (kontrollieren = to check); to become ≠ bekommen → „werden“; sensible ≠ sensibel → „sinnvoll/vernünftig“ (sensitive data → „sensible Daten“ ✓); provision ≠ Provision (= commission) → „bereitstellen/Bereitstellung“; note ≠ Note (= grade) → „Hinweis/Anmerkung“; billion ≠ Billion → „Milliarde“; consequent ≠ konsequent → „nachfolgend“; to handle ≠ handeln → „verarbeiten/behandeln“; map ≠ Mappe → „Zuordnung/Abbildung“; to spend ≠ spenden → „ausgeben“; brave ≠ brav → „mutig“; gift ≠ Gift (= poison) → „Geschenk“; chef ≠ Chef (= boss) → „Koch“. + +## 10. Reviewer checklist — German MT failure modes + +1. „du“/„dein“ anywhere, or a register switch mid-page. +2. English title case in headings, or a heading rendered as a full Sie-sentence where an Infinitiv belongs. +3. Deppenleerzeichen and missing Durchkopplung („Bare Metal Server“, „Control Plane Knoten“, „Kubernetes Cluster“). +4. Wrong or fluctuating gender for the same loanword; wrong plural („die Clusters“, „die Containers“); missing genitive `-s`. +5. Decimal points left in prose, or decimal commas wrongly injected into versions, IPs, image tags or code. +6. Em dash — or straight `"` instead of – and „…“. +7. Missing comma before dass/weil/wenn/relative clauses or before an extended infinitive group. +8. English word order surviving: verb not in second position, subordinate-clause verb not final, infinitive not at the end after a modal. +9. Untranslated leftovers — or the opposite: translated API kinds, flags, field names, or do_not_translate terms. +10. One source term rendered several ways on the same page; a glossary term overridden by an improvised synonym. +11. Calques from §8, especially „erlaubt Ihnen zu“, „in diesem Artikel werden wir“, filler „einfach“. +12. False friends from §9 — check every occurrence of „eventuell“, „aktuell“, „kontrollieren“, „sensibel“. +13. Sentences over ~25 words with several nested subordinate clauses: split them. German technical prose tolerates them worse than the English source suggests. diff --git a/hack/i18n/style-guides/es.md b/hack/i18n/style-guides/es.md new file mode 100644 index 000000000..1b2df25d5 --- /dev/null +++ b/hack/i18n/style-guides/es.md @@ -0,0 +1,89 @@ +Applies to both writing and reviewing: every rule below is also a review checkpoint. Code blocks, inline `code`, Hugo shortcodes, URLs, file paths and front-matter keys are masked by the pipeline (`§§NAME_N§§`) — never translate, reorder or "fix" them; the punctuation and number rules below apply to PROSE ONLY. + +## 1. Register and variety + +- Neutral international Spanish, usable in Spain and Latin America: no region-specific slang, no `vosotros`, no `vos`, and avoid the `ordenador`/`computadora` split (say `equipo`, `servidor`, `nodo`). +- Address the reader with **tú**, consistently on every page. Imperatives in `tú`: ✓ `Ejecuta`, `Crea`, `Consulta` — ✗ `Ejecute`, `Realicen`. A single `Consulte la guía` inside a `tú` page is a major finding. +- Docs: precise, imperative, short sentences, no first person except `En esta guía…`. Blog/landing: may transcreate, may use `nosotros` for the Cozystack team (`hemos publicado`), rhetorical questions allowed. +- Drop English politeness padding: `Please run…` → ✓ `Ejecuta…` — ✗ `Por favor, ejecuta…`. +- Gender-neutral wording by rephrasing (`el equipo`, `quienes administran el clúster`), never `@`, `x`, `e` or `usuarios/as`. + +## 2. Headings and capitalization — CRITICAL + +- Spanish uses **sentence case**, never English title case. `Getting Started with Cozystack` → ✓ `Primeros pasos con Cozystack` — ✗ `Primeros Pasos Con Cozystack`. +- Only the first word plus proper nouns, brands and API kinds are capitalized: ✓ `Cómo funciona el almacenamiento en LINSTOR`. +- `-ing` headings become a noun or an infinitive, as on kubernetes.io/es: `Installing Cozystack` → ✓ `Instalación de Cozystack` / `Instalar Cozystack` — ✗ `Instalando Cozystack`. Keep one pattern per page. +- Question headings take both marks: `How does it work?` → ✓ `¿Cómo funciona?`. +- No capital after a colon: ✓ `Nota: el clúster debe estar en ejecución.` — ✗ `Nota: El clúster…`. Capitalize after a colon only before a full quotation. +- Lowercase months, days, languages and nationalities: ✓ `el lunes`, `julio`, `en inglés`. +- Table headers and UI labels also follow sentence case. Do not drop the article for headline-ese: ✓ `Instalar el clúster` — ✗ `Instalar clúster`. + +## 3. Terminology policy (a do_not_translate + preferred list is injected separately — apply it first) + +Decision rule for a term in NEITHER list, in this order: +1. Is it an identifier the reader must match in a manifest, CLI or UI (`kubectl`, `namespace` inside a command, flags, field names, API kinds)? → keep verbatim, unchanged. +2. Does kubernetes.io/es or the CNCF Cloud Native Glossary already have a Spanish rendering? → use it. +3. Is there a natural, unambiguous Spanish word an engineer actually says? → translate (`rendimiento`, `red`, `almacenamiento`, `copia de seguridad`, `registros`). +4. Otherwise keep the English term bare and gloss it once, on first use: ✓ `sidecar (contenedor auxiliar)`, then `sidecar` alone. Never invent a translation nobody uses. +- Pick one rendering per term and keep it identical across the whole page. +- **API kinds in prose** keep the English name, the capital and no accent: ✓ `el Pod`, `el Deployment`, `los Secrets`, `el Node`, `el Tenant`. The same word as an ordinary concept is lowercase and Spanish: ✓ `un despliegue continuo`, `los nodos del clúster`, `el inquilino (tenant)`. Never inflect a kept term: ✗ `Deploymentos`, ✗ `kuberneteses`, ✗ `el Helm's chart`. +- **Gender** = gender of the implicit Spanish hypernym: el pod, el clúster, el nodo, el contenedor, el chart, el endpoint, el backend, el volumen, el despliegue, el almacenamiento, el espacio de nombres; la imagen, la red, la carga de trabajo, la API, la caché, la instancia, la máquina virtual. Agreement must follow: ✓ `una imagen firmada`, `los pods pendientes` — ✗ `la pod`, `el imagen`, `los clúster`. +- **Plurals**: adapted words take Spanish plurals (`clústeres`, `contenedores`, `servidores`); bare loanwords take `-s` (`los pods`, `los charts`, `los endpoints`). Acronyms are invariable: `las API`, `los CRD`. +- **Accent only when the adaptation is established** (RAE plus real ecosystem use): ✓ `clúster`, `búfer`, `caché`, `módulo`. Otherwise leave the loanword bare: ✓ `pod`, `chart`, `sidecar` — ✗ `pód`, ✗ `sídecar`. Do not italicize ecosystem loanwords. + +## 4. Typography and punctuation + +- Inverted marks are mandatory and go **where the question or exclamation actually starts**, not at the start of the sentence: ✓ `Si el nodo falla, ¿qué pasa con los datos?`, ✓ `Listo, ¡ya tienes un clúster!` — ✗ `¿Si el nodo falla, qué pasa con los datos?`, ✗ `Qué pasa con los datos?`. +- Quotes: use **"comillas inglesas"** (straight double quotes), not «comillas latinas». Rationale: kubernetes.io/es and the wider Latin American tech web use them, `«»` reads as Spain-specific and is inconsistent with the English source and with Markdown tooling. Nest as `"… 'anidado' …"`. Prefer **bold** over quotes for UI labels. +- **Raya (—)** for parentheticals, with a space outside and none inside: ✓ `El clúster —ya en ejecución— acepta cargas nuevas.` Never use `-` or `–` for this, and never leave a stray closing raya. In blog dialogue the raya opens the line with no space after it: `—Empezamos con tres nodos.` +- **No Oxford comma**: Spanish takes no comma before the `y`/`o` that closes a list. ✓ `nodos, discos y redes` — ✗ `nodos, discos, y redes`. (Exception only to prevent a real ambiguity.) +- Comma before an adversative or after a fronted subordinate clause: ✓ `Si el pod no arranca, revisa los eventos.`, ✓ `Funciona, pero requiere más memoria.` Never between subject and verb: ✗ `El operador de almacenamiento, crea el volumen.` +- Suspension points are always three, followed by a space. Use `:` rather than `;` to introduce lists and code blocks. + +## 5. Numbers, dates, units + +- **Decimal comma in prose**: `3.14 seconds` → ✓ `3,14 segundos` — ✗ `3.14 segundos`. NEVER convert inside version numbers, code, CLI output, YAML values, file names or CIDR notation: ✓ `Cozystack v1.5.0`, `10.0.0.1/24`, `cpu: 0.5`. +- Thousands: never a comma. `10,000 pods` → ✓ `10 000 pods` (non-breaking space) or `10000`; four digits stay unseparated (`1024`). +- Dates: `July 24, 2026` → ✓ `24 de julio de 2026` (lowercase month, `de` twice); numeric form `24/07/2026`. Never `Julio 24, 2026`. Front-matter `date:` values are masked — leave them. +- Space between number and unit, and before `%`: ✓ `16 GB`, `4 vCPU`, `500 ms`, `25 %`. Keep unit symbols in English/SI (`GB`, `TB`, `GiB`, `ms`). +- Abbreviations follow Spanish forms: `1.º`, `n.º`, `EE. UU.`. Spell out `segundos`, `minutos` in prose unless quoting a metric. + +## 6. Grammar traps when translating from English + +- **Gerund** — the top calque. Never gerund-as-adjective or gerund-of-consequence: ✗ `un archivo conteniendo la configuración` → ✓ `un archivo que contiene la configuración`; ✗ `El nodo falló, provocando un reinicio` → ✓ `El nodo falló y provocó un reinicio`. `Using X, you can…` → ✓ `Con X puedes…`. +- **Passive → pasiva refleja**, which Spanish strongly prefers: ✗ `El clúster es creado por el operador` → ✓ `El operador crea el clúster` / `El clúster se crea automáticamente`; ✗ `Los datos son almacenados en…` → ✓ `Los datos se almacenan en…`. +- **`you can`**: `puedes` when it is a real option, `es posible` in neutral docs — and often just drop it: `You can find the logs in /var/log` → ✓ `Los registros están en /var/log`. +- **Possessives → definite article**: `your cluster` → ✓ `el clúster` (`tu clúster` only when contrasting with someone else's); `Deploy your application` → ✓ `Despliega la aplicación`; `its configuration` → ✓ `la configuración`. +- **Noun stacking**: never a chain of four `de`. `Kubernetes cluster node pool autoscaling configuration` → ✗ `configuración de autoescalado de grupos de nodos de clúster de Kubernetes` → ✓ `configuración del autoescalado de los grupos de nodos en Kubernetes`. Break chains with an adjective (`red virtual`, `disco local`), a preposition (`en`, `para`) or a relative clause. Two consecutive `de` maximum. +- **ser vs estar**: identity or definition → `ser` (`Cozystack es una plataforma`); state, result, availability, location → `estar` (`el pod está listo`, `el servicio está disponible`, `el clúster está en ejecución`). ✗ `el pod es listo`. +- **Subjunctive** after `cuando` with future reference and after `asegúrate de que`: ✓ `Cuando el pod esté listo, ejecuta…` — ✗ `Cuando el pod está listo…`; ✓ `Asegúrate de que el nodo tenga suficiente memoria.` But `si` + present takes the indicative: ✓ `Si quieres aislar la carga…` — ✗ `Si quieras…`. +- Avoid the bureaucratic anaphoric `el mismo/los mismos`: ✗ `Descarga el chart y aplica el mismo` → ✓ `Descarga el chart y aplícalo`. + +## 7. Translationese and calques to fix + +- Filler openers: ✗ `En este artículo vamos a ver cómo…` → ✓ `Esta guía explica cómo…`, or start straight with the content. Delete `simplemente`, `básicamente`, `de hecho`, `es importante notar que` unless the English carries real emphasis. +- `support` → ✓ `admitir`, `ser compatible con`, `permitir` — ✗ `soportar` (that means "to endure"). `supported versions` → ✓ `versiones compatibles`; `unsupported` → ✓ `no compatible`. +- `apply`: ✓ `aplicar un manifiesto`, ✓ `esto se aplica a…`; but `apply a role` → ✓ `asignar un rol` and `apply for` → ✓ `solicitar` — ✗ `aplicar para`. +- `remove` → ✓ `eliminar`, `quitar` — ✗ `remover` (= to stir). `encrypt` → ✓ `cifrar`, `el cifrado` — ✗ `encriptar`. `library` → ✓ `biblioteca` — ✗ `librería`. `customize` → ✓ `personalizar` — ✗ `customizar`. +- Others: `deprecated` → `obsoleto`/`en desuso`; `performance` → `rendimiento`; `issue` → `problema`/`incidencia`; `feature` → `funcionalidad`; `release` → `versión`/`lanzamiento`; `setup` (n.) → `configuración`; `reset` → `restablecer`; `access` (v.) → `acceder a` (✗ `accesar`); `assume` → `suponer` (✗ `asumir`); `in order to` → `para` (✗ `en orden de`); `make sense` → `tener sentido` (✗ `hacer sentido`); `based on` → `según`/`a partir de` when `basado en` piles up; `additionally` → `además`. +- No English-style hyphenated compounds: ✗ `alta-disponibilidad`, ✗ `multi-inquilino` → ✓ `alta disponibilidad`, `multiinquilino`. No English possessive `'s`. +- Link text: never `haz clic aquí` → ✓ `Consulta la guía de instalación`. + +## 8. False friends (tech and business) + +`actually` → `en realidad`/`de hecho` (✗ `actualmente` = currently) · `eventually` → `con el tiempo`/`finalmente` (✗ `eventualmente` = occasionally) · `library` → `biblioteca` · `support` → `admitir` · `realize` → `darse cuenta` (✗ `realizar` = to carry out) · `sensible` → `sensato`/`razonable` (but `sensitive data` → `datos confidenciales`) · `assist` → `ayudar` (✗ `asistir` = to attend) · `consistent` → `coherente`/`uniforme` (✗ `consistente` = thick, solid) · `resume` → `reanudar` (✗ `resumir` = to summarize) · `topic` → `tema` (✗ `tópico` = cliché) · `large` → `grande` (✗ `largo` = long) · `ultimately` → `en última instancia` (✗ `últimamente` = lately) · `commodity hardware` → `hardware estándar` · `abort` → `cancelar`/`interrumpir` · `exit` → `salir` (✗ `éxito`) · `discrete` → `discreto` vs `discreet` → `prudente`. `argument`, `instance` and `policy` are fine as `argumento`, `instancia`, `política` in technical senses. + +## 9. Reviewer checklist — frequent MT failures in Spanish + +1. English Title Case leaking into headings, table headers or bold labels. +2. Missing opening `¿`/`¡`, or the mark placed at the sentence start instead of where the question begins. +3. Decimal point left as `.` in prose — or, worse, a decimal comma injected into a version, IP, CIDR or YAML value. +4. `usted`/`vosotros` drift, or mixed imperatives (`ejecuta` … `ejecute`) on one page. +5. Agreement drift around loanwords: `la pod`, `el imagen`, `los clúster`, `las nodos`. +6. Gerund calques (`conteniendo`, `permitiendo`, `resultando en`) and English passives left as `es/son` + participle. +7. `soportar`, `librería`, `encriptar`, `remover`, `customizar`, `accesar`, `en orden de`. +8. Missing article (`Instalar clúster`) or `de`-chains of four or more. +9. Capital after a colon; capitalized months or days; `Por favor` retained. +10. Terminology inconsistency: one English term rendered two ways on the same page, or a glossary/do_not_translate term inflected or translated. +11. Placeholders `§§NAME_N§§` lost, duplicated or reordered; code, URLs or YAML keys translated. +12. English clause order preserved in over-long sentences, comma splices, `el cual`/`el mismo` overuse, literal `aquí` link text. diff --git a/hack/i18n/style-guides/hi.md b/hack/i18n/style-guides/hi.md new file mode 100644 index 000000000..93de8db1a --- /dev/null +++ b/hack/i18n/style-guides/hi.md @@ -0,0 +1,100 @@ +# Hindi (hi) — style guide + +Modern standard Hindi (Devanagari) as written by Indian tech authors. Hinglish is normal and expected: keep widely-used English technical terms as English terms rather than forcing Sanskritized coinages. Translate the connective prose, not the jargon. Register: professional, clear, never bureaucratic (सरकारी हिंदी), never slangy. + +## 1. Register and address + +- Address the reader as **आप** only. Never तू/तुम. Verbs take the आप form: करें, चलाएँ, देखें, कर सकते हैं. +- Imperative style: use the **-एँ** form consistently (चलाएँ, बनाएँ, जोड़ें). Do not mix in कीजिए/कीजिये or करो within a page. +- Docs: terse, instructional, present tense. Blog: same person, slightly warmer; the project may speak as हम. +- No exclamation marks, no rhetorical questions, no emoji, no filler (आइए जानते हैं, तो चलिए शुरू करते हैं). +- EN "You can now access the dashboard." → ✗ "अब आप डैशबोर्ड को एक्सेस कर सकते हैं!" → ✓ "अब आप dashboard खोल सकते हैं।" + +## 2. How much English to keep — the central rule + +Test each term with one question: **would an Indian DevOps engineer say this word in English while speaking Hindi?** If yes, it stays an English word. Then decide the script by bucket: + +1. **Latin script, verbatim** — anything the reader must type, match, or click: product and project names (Cozystack, Kubernetes, Talos, Flux), CLI names and flags (kubectl, --namespace), API kinds and fields (Pod, Deployment, Secret, Node, ConfigMap, StorageClass, spec.replicas), file names, env vars, UI labels, acronyms (API, CNI, CSI, RBAC, TLS, YAML, GPU). +2. **Devanagari transliteration of the English word** — only for the general infra nouns the injected glossary assigns a Hindi rendering (e.g. cluster/node/storage as ordinary nouns). Use the glossary form exactly; never invent new transliterations for terms it does not list. +3. **Real Hindi** — everything else: verbs, adjectives, connectors, and ordinary nouns (उपलब्ध, कॉन्फ़िगर करें, इसके बाद, ध्यान दें, अनुमति, आवश्यकता). + +Default for a term not covered by the glossary: **keep it in Latin.** Never coin a Devanagari equivalent for infra jargon. + +- ✗ संगणक, कुंजीपटल, अभिकलन, स्मृति, संचिका, जालक्रम, कूटशब्द, आभासी संगणित्र → ✓ computer, keyboard, compute, memory, file, network, password, VM +- EN "Configure the ingress controller and the load balancer." → ✗ "प्रवेश नियंत्रक और भार संतुलक को विन्यस्त करें।" → ✓ "ingress controller और load balancer कॉन्फ़िगर करें।" +- EN "Kubernetes schedules the pod on a node." → ✗ "कुबेरनेट्स पॉड को एक नोड पर शेड्यूल करता है।" (brand transliterated) → ✓ "Kubernetes पॉड को नोड पर schedule करता है।" +- Over-translation is as bad as under-translation: EN "high availability" → ✗ "उच्च उपलब्धता" is acceptable only if the glossary says so; otherwise keep "high availability". + +## 3. Headings + +- Devanagari has no letter case — there is nothing to title-case. Do not capitalize or restyle; just write a natural phrase. +- No terminal । in headings. No trailing colon. +- English terms inside a heading keep exactly the form used in the body — same script, same capitalization. ✗ heading "क्लस्टर इंस्टॉल करना" + body "cluster" (script drift). +- Prefer verbal nouns or short noun phrases, consistently within a page: EN "Installing Cozystack" → ✓ "Cozystack इंस्टॉल करना"; EN "Prerequisites" → ✓ "आवश्यक शर्तें"; EN "Troubleshooting" → ✓ "समस्या निवारण". + +## 4. Terminology policy (terms outside the glossary) + +Decision order: glossary `do_not_translate` → verbatim Latin; glossary `preferred` → that exact rendering, every occurrence; otherwise apply §2 and default to Latin. + +- **API kinds stay Latin and singular in prose**: "Pod को delete करें", "Deployment में replicas बढ़ाएँ", "यह Secret namespace में बनता है". Never transliterate a kind (✗ डिप्लॉयमेंट ऑब्जेक्ट for `kind: Deployment`). +- Mirror the glossary's Tenant/tenant split: capitalized API kind (`Node`, `Tenant`) → Latin; the same word as an ordinary noun (a node, a tenant) → the glossary's Devanagari form. +- **First mention**: when a Devanagari-rendered term first appears on a page, gloss it once — "टेनेंट (tenant)" — then use the plain form. Do not repeat the gloss. +- One term = one rendering per page and per docs version. If you chose Latin once, it is Latin everywhere on that page. +- Never attach Devanagari inflection to a Latin word: ✗ Podों, ✗ Deploymentस, ✗ नोडnode. Express plurality with a quantifier: ✓ "कई Pod", "सभी Node", "दो PVC". + +## 5. Script and typography + +- End full sentences in prose with **पूर्ण विराम ।**, including sentences that end on a Latin word: ✓ "इसके बाद `kubectl apply` चलाएँ।" Never write ".।". Never place । inside code, headings, or list fragments that are not sentences. +- Keep the period for version numbers, decimals, abbreviations and anything inside code (v1.2.5). +- Use straight double quotes " " for quoted UI strings and values; keep the quoted English string in Latin. Avoid mixed ‘ ’/“ ” pairs. +- One space between a Devanagari run and an adjacent Latin/number run; no space before ।, comma, or a closing bracket. Postpositions after Latin words are separated by a space: "Node पर", "namespace में". +- Never mix scripts inside a single word, and never transliterate identifiers, flags, paths, or commands. + +## 6. Numbers, dates, units + +- **Western Arabic digits only** (1, 2, 3). Never Devanagari digits (१, २, ३). +- Use **international grouping** — 1,000 / 1,000,000. Do **not** use Indian grouping (✗ 10,00,000) and never convert to lakh/crore in technical content. +- Decimal point is `.` — ✓ 1.5 GB. +- Dates: "24 जुलाई 2026" in prose; ISO `2026-07-24` when the source is ISO. Never MM/DD/YYYY. +- Units keep Latin symbols with a space: 16 GB, 500 ms, 4 vCPU, 10 Gbit/s. Percent has no space: 50%. Ranges: "2–4 GB" or "2 से 4 GB". + +## 7. Grammar traps (EN → HI) + +- **SOV order.** EN "Cozystack deploys the application to the cluster." → ✗ "Cozystack डिप्लॉय करता है application को क्लस्टर में।" → ✓ "Cozystack application को क्लस्टर में deploy करता है।" The verb goes last. +- **Gender of English loanwords.** Consonant-final loans are **masculine**: क्लस्टर, नोड, पॉड, सर्वर, नेटवर्क, स्टोरेज, बैकअप, प्लेटफ़ॉर्म. Feminine: फ़ाइल, मशीन, सर्विस, इमेज, and -ी endings (मेमोरी, डायरेक्टरी, लाइब्रेरी). Keep gender consistent across the whole page: ✗ "क्लस्टर बनाई गई" → ✓ "क्लस्टर बनाया गया"; ✗ "फ़ाइल बनाया गया" → ✓ "फ़ाइल बनाई गई". +- **का/की/के agreement** follows the *possessed* noun: ✓ "क्लस्टर की स्थिति", "नोड का नाम", "Pod के logs". +- **Postpositions.** in → में, on → पर, from → से, for → के लिए. ✗ "क्लस्टर के लिए deploy करें" for "deploy to the cluster" → ✓ "क्लस्टर में deploy करें". +- **Ergative ने** only with perfective transitive verbs: ✓ "हमने क्लस्टर बनाया"; ✗ "Cozystack ने क्लस्टर बनाता है" (imperfective — no ने); ✗ ने with intransitives ("Pod ने शुरू हुआ"). +- **Compound verbs** read naturally; do not strip them: ✓ "config फ़ाइल लिख दें", "logs देख लें" where the source implies completion. Do not add them where the source is neutral. +- Honorific verb forms must stay consistent — do not switch between "करें" and "करते हैं" for the same instructional voice. + +## 8. Translationese and calque patterns to fix + +- **"आप ... कर सकते हैं" everywhere.** In step-by-step docs use the imperative. EN "You can create a tenant with kubectl." → ✗ "आप kubectl का उपयोग करके एक टेनेंट बना सकते हैं।" → ✓ "kubectl से टेनेंट बनाएँ।" +- **एक as an article.** Hindi has no indefinite article. ✗ "एक क्लस्टर बनाएँ" → ✓ "क्लस्टर बनाएँ". Keep एक only when the count matters. +- **Literal passive.** EN "The cluster is created by the operator." → ✗ "क्लस्टर ऑपरेटर के द्वारा बनाया जाता है।" → ✓ "operator क्लस्टर बनाता है।" +- **English-order relative clauses.** EN "the node that runs the workload" → ✗ "नोड जो वर्कलोड चलाता है" → ✓ "वर्कलोड चलाने वाला नोड" (or जो … वह with correct correlative). +- **"का उपयोग करके" / "के माध्यम से" stacked in every sentence** → prefer "से": ✓ "Helm से इंस्टॉल करें". +- **"यह सुनिश्चित करें कि" / "कृपया"** in every step — drop them; docs are already polite via आप. +- Word-by-word renderings of English idiom ("out of the box" → ✗ "डिब्बे से बाहर") → ✓ "बिना अतिरिक्त कॉन्फ़िगरेशन के" or keep "out of the box". + +## 9. Reviewer checklist (MT failure modes) + +1. Sanskritized coinage anywhere (संगणक, कुंजीपटल, अभिकलन, स्मृति, संचिका) → reject. +2. Brand/product transliterated (कुबेरनेट्स, कोज़ीस्टैक, कुबेक्टल) → must be Latin. +3. Same term in two scripts or two spellings on one page (cluster vs क्लस्टर, स्टोरेज vs भंडारण). +4. Gender drift on loanwords across paragraphs; का/की/के disagreement. +5. English SVO order preserved; verb not final. +6. Wrong postposition, or में/पर/के लिए swapped. +7. ने inserted with imperfective or intransitive verbs. +8. Passive constructions copied from English. +9. "आप … कर सकते हैं" or "एक" repeated in nearly every sentence. +10. Devanagari digits, Indian digit grouping, lakh/crore, MM/DD dates. +11. Missing or doubled ।; । after a heading; । inside code. +12. Devanagari suffixes glued to Latin words (Podों), or a translated flag/identifier (`--नाम`). +13. Mixed imperative registers (चलाएँ vs चलाइए vs चलाओ) in one page. +14. Terminology diverging from the injected glossary. + +## 10. Pipeline note + +Code blocks, inline code, Hugo shortcodes, URLs, HTML attributes and front-matter keys are masked structurally by the pipeline. Never translate, reorder, transliterate or re-space anything inside a mask; front-matter `title` and `description` values are translated, their keys are not. diff --git a/hack/i18n/style-guides/pt-br.md b/hack/i18n/style-guides/pt-br.md new file mode 100644 index 000000000..30ebf2195 --- /dev/null +++ b/hack/i18n/style-guides/pt-br.md @@ -0,0 +1,80 @@ +Brazilian Portuguese (pt-BR), never European Portuguese. Follow kubernetes.io pt-BR localization conventions and the CNCF Cloud Native Glossary. Natural BR phrasing beats literal calques: if a sentence can only be parsed by reconstructing the English, rewrite it. Code blocks, inline `code`, Hugo shortcodes, URLs, front-matter keys and §§PLACEHOLDERS§§ are masked by the pipeline — never translate or reorder them. + +## 1. Register and address + +- Always **você** (explicit or implicit), consistently within a page. Never *tu*, *vós*, *o senhor*, and never mix conjugations: ✗ "se tu quiseres", ✗ "podes ver" → ✓ "se você quiser", "você pode ver". +- Friendly-professional. Docs: imperative, short sentences, one instruction per sentence ("Execute o comando", "Verifique o status"). Drop English politeness: "Please run" → ✗ "Por favor, execute" → ✓ "Execute". +- Blog/landing: transcreate. First person plural is fine ("lançamos", "nossa equipe"); avoid hype adjectives — *powerful* → ✗ "poderoso" → ✓ "robusto", "eficiente", or drop. +- The product takes an article in running prose: ✓ "o Cozystack instala…", "no Kubernetes"; omit it in labels and titles. + +## 2. Headings and capitalization (critical) + +- Portuguese uses **sentence case**, never English title case: capitalize only the first word and proper nouns. "Installing the Storage Backend" → ✗ "Instalando o Backend de Armazenamento" → ✓ "Instalação do backend de armazenamento". +- Prefer a noun phrase (or infinitive) over the English gerund heading, consistently within a page: "Configuring Networking" → ✗ "Configurando a Rede" → ✓ "Configuração de rede" / "Configurar a rede". +- Do not capitalize common nouns mid-sentence because English did: "the Cluster and the Node" → ✓ "o cluster e o nó". +- Same rule for `title`/`description` front matter, table headers, alt text, button and menu labels. + +## 3. Terminology policy (a glossary is injected separately — do not restate it) + +- Decision rule for terms **not** in the glossary, in order: (1) kubernetes.io pt-BR translates it → use that; (2) the CNCF Glossary pt-BR has it → use that; (3) a BR platform engineer would say the English word out loud in a standup → keep English; (4) otherwise translate, with the English in parentheses on first use only. +- Kept in English (real BR usage): pod, cluster, deploy, container, namespace, commit, pull request, backup, snapshot, endpoint, cache, log, bucket, overhead, hypervisor, bare metal, load balancer. +- Translated: workload → carga de trabalho, node → nó, storage → armazenamento, network → rede, image → imagem, file → arquivo, feature → recurso, wizard → assistente, release → versão/lançamento. +- **API objects**: when the word names a Kubernetes/Cozystack API kind, keep it capitalized and in English — "o Pod", "um Deployment", "o Secret", "o objeto Node", "o Tenant" (must match `kind:` in the manifest). When it is the everyday concept, translate and lowercase: "os pods do cluster", "os nós do cluster", "os segredos da aplicação". +- **Gender**: masculine by default for English loanwords (o pod, o cluster, o deploy, o container, o backup, o node, o namespace, o endpoint); feminine when the underlying PT head noun is feminine (a imagem, a API, a VM, a URL, a tag, a role). ✗ "a cluster", ✗ "o VM", ✗ "a pod". +- **Plural**: inflect with -s, no apostrophe — os pods, os clusters, os deployments, os namespaces, as VMs, as APIs, os PVCs. ✗ "os pod", ✗ "as VM's". Adjectives agree: "pods prontos", "cargas de trabalho críticas". + +## 4. Typography and punctuation + +- Quotation marks: **straight double quotes** `"…"`, nested `'…'`. Do not use «guillemets» (pt-PT/literary) or curly quotes — the docs are code-adjacent and quoted strings get copied into shells. +- **Travessão (—)** with spaces for parenthetical asides; keep the source's em dashes, never downgrade them to a hyphen: ✓ "O tenant — a unidade de isolamento — recebe seu próprio namespace." +- **No Oxford comma.** Portuguese forbids a comma before *e*/*ou* closing an enumeration: "storage, networking, and compute" → ✗ "armazenamento, rede, e computação" → ✓ "armazenamento, rede e computação". +- Comma after a fronted adverbial or subordinate clause: ✓ "Depois de instalar o Cozystack, aplique o manifesto." Never a comma between subject and verb, however long the subject: ✗ "O cluster com três nós de controle, requer…". +- **Crase (à)** — verify every occurrence. Use it when preposition *a* + feminine article *a* merge: ✓ "acesso à API", "conecte-se à rede", "à medida que", "devido à latência", "graças à replicação". No crase before masculine nouns, verbs or article-less plurals: ✗ "à partir de" → ✓ "a partir de"; ✗ "à fim de" → ✓ "a fim de"; ✗ "acesso à documentos" → ✓ "acesso a documentos"; ✓ "referente ao cluster". +- Use "e", never "&". Keep `%`, `/` and CLI punctuation exactly as in the source. + +## 5. Numbers, dates, units + +- **Decimal comma in prose**: "3.14" → ✓ "3,14"; "0.5 vCPU of overhead" → ✓ "0,5 vCPU de overhead". Thousands separator is a period: "1,000 nodes" → ✓ "1.000 nós"; "12,500" → ✓ "12.500". +- **Never** convert inside code, CLI output, YAML values, Kubernetes quantities, CIDRs or version numbers: `v1.2.3`, `500m`, `10.0.0.0/8` stay exactly as written — including when quoted in prose. A bare decimal in ordinary prose (the "0.5 vCPU" case above) follows the prose rule, not this one. +- Dates: "July 24, 2026" → ✓ "24 de julho de 2026" (month lowercase); numeric 24/07/2026; ranges "de 2024 a 2026". 24-hour clock: "9 AM UTC" → ✓ "9h UTC" / "09:00 UTC". +- Units: space between number and unit (10 GB, 500 ms, 2 vCPU, 4 GiB); no space before `%` (50%); keep byte/rate units in standard casing (GiB, TiB, Mbps). + +## 6. Grammar traps translating from English + +- **Gerundismo / -ing calque** — the notorious BR error. English -ing → infinitive or noun; never "estar + gerúndio" for a future or habitual action: ✗ "vamos estar enviando o manifesto" → ✓ "vamos enviar o manifesto"; "Before installing, check the requirements" → ✗ "Antes de instalando" → ✓ "Antes de instalar, verifique os requisitos". Genuine progressives are fine: ✓ "o pod está iniciando". +- **Passive**: agentless English passive → active imperative or synthetic passive with *se*: "The manifest is applied automatically" → ✓ "Aplica-se o manifesto automaticamente" / ✓ "O Cozystack aplica o manifesto automaticamente". In procedures, prefer the imperative. +- "you can" → alternate "você pode" with impersonal "é possível" so a page does not repeat "você pode" ten times. +- **Possessives → definite article**: "Open your terminal and check your cluster" → ✗ "Abra o seu terminal e verifique o seu cluster" → ✓ "Abra o terminal e verifique o cluster". +- **Noun stacking → de-chains, max two links**: "Kubernetes cluster node pool autoscaling configuration" → ✗ "configuração do autoscaling do pool de nós do cluster Kubernetes" → ✓ "configuração do autoscaling para pools de nós (cluster Kubernetes)". +- **ser vs estar**: definitional → *ser* ("O Cozystack é uma plataforma"); state/condition → *estar* ("O cluster está pronto", "o pod está em execução"). ✗ "O cluster é pronto". +- **Future subjunctive after quando/se/assim que**: "When you create the tenant" → ✗ "Quando você cria o tenant" → ✓ "Quando você criar o tenant"; "If the pod fails" → ✓ "Se o pod falhar"; "as soon as it is ready" → ✓ "assim que estiver pronto". +- **Colocação pronominal (BR = próclise)**: ✗ "Não aplica-se a clusters legados" → ✓ "Não se aplica a clusters legados". Never open a sentence with an unstressed pronoun; avoid pt-PT mesoclisis ✗ "permitir-lhe-á" → ✓ "vai permitir que você". +- **Future tense**: BR prefers "vai + infinitivo" or the simple present: "This will create a namespace" → ✗ "Isto irá criar um namespace" → ✓ "Isso cria um namespace". + +## 7. Translationese and calques to eliminate + +- "In this article, we will…" → ✗ "Neste artigo, nós vamos…" → ✓ "Veja como…" / "Este guia mostra…". +- Delete filler — *simply*, *just*, *easily*: ✗ "simplesmente execute" → ✓ "execute". +- support → ✗ "suportar" (= to endure) → ✓ "oferecer suporte a", "ser compatível com", "aceitar": "Cozystack supports GPU passthrough" → ✓ "O Cozystack oferece suporte a GPU passthrough". +- application → ✓ "aplicação" (server-side workloads); "aplicativo" only for mobile/desktop apps. +- encrypt → ✗ "encriptar" → ✓ "criptografar" / "criptografia". library → ✗ "livraria" → ✓ "biblioteca". customize → ✗ "customizar" → ✓ "personalizar". delete → ✗ "deletar" → ✓ "excluir"/"remover". set → ✗ "setar" → ✓ "definir"/"configurar". address (an issue) → ✗ "endereçar" → ✓ "tratar"/"resolver"/"corrigir". +- requirement → ✗ "requerimento" (= a petition) → ✓ "requisito". performance → ✓ "desempenho". check → ✓ "verificar" (not "checar"). issue → ✓ "problema" (keep "issue" only for a GitHub issue). +- "Once you have installed…" → ✗ "Uma vez que você instalou" (= *because*) → ✓ "Depois de instalar" / "Assim que instalar". "Assuming that…" → ✓ "Supondo que". "in order to" → ✓ "para" (not "de forma a"). "make sure" → ✓ "certifique-se de que"; "note that" → ✓ "observe que"; "keep in mind" → ✓ "lembre-se de que". + +## 8. False friends (EN → PT) + +actually → ✓ "na verdade" (✗ atualmente = *currently*) · eventually → ✓ "por fim", "com o tempo" (✗ eventualmente = *occasionally*) · library → ✓ biblioteca · support → ✓ oferecer suporte a · realize → ✓ "perceber", "notar" (✗ realizar = *to carry out*) · pretend → ✓ "fingir" (✗ pretender = *to intend*) · push (git) → keep "push" or ✓ "enviar" (✗ "empurrar") · assume → ✓ "supor", "presumir" (✗ assumir = *to take on*) · resume → ✓ "retomar" (✗ resumir = *to summarize*) · comprehensive → ✓ "abrangente" (✗ compreensivo = *understanding*) · sensible → ✓ "sensato" (✗ sensível = *sensitive*) · attend → ✓ "participar de" (✗ atender = *to serve*) · exit → ✓ "sair" (✗ êxito = *success*) · policy → ✓ "política" (never "polícia") · parent (resource) → ✓ "pai"/"superior". + +## 9. Reviewer checklist — pt-BR machine-translation failure modes + +1. **European Portuguese leaking in** — reject on sight: ficheiro→arquivo, utilizador→usuário, ecrã→tela, rato→mouse, aceder a→acessar, guardar→salvar, apagar→excluir, equipa→equipe, registo→registro, facto→fato, contacto→contato, "está a executar"→"está executando", connosco→conosco, gestor→gerenciador, telemóvel→celular. +2. English **title case** surviving in headings, `title`/`description`, table headers or link text. +3. Decimal points left as "3.14"/"1,000" in prose — or commas wrongly injected into code, versions, CIDRs. +4. Missing or hypercorrect **crase** ("à partir de", "acesso a API"). +5. Comma before *e*/*ou*, or a comma splitting subject from verb. +6. Gender/number disagreement on loanwords ("a cluster", "os pod", "VM's"). +7. Gerundismo, "irá + infinitivo" future, enclisis after negation ("não aplica-se"). +8. Literal "suportar", "customizar", "deletar", "encriptar", "endereçar", "livraria", "uma vez que", "simplesmente". +9. Register drift: *tu*/*você* mixing, "por favor" in imperatives, marketing hype in docs, stiff formality in blog. +10. Untranslated or dropped sentences; translated API kinds, CLI flags, YAML keys or URLs — all hard failures. +11. Glossary terms rendered inconsistently across a page (e.g. "nó" in one paragraph, "node" in the next). diff --git a/hack/i18n/style-guides/ru.md b/hack/i18n/style-guides/ru.md new file mode 100644 index 000000000..df4fd6c31 --- /dev/null +++ b/hack/i18n/style-guides/ru.md @@ -0,0 +1,96 @@ +Этот файл — одновременно инструкция переводчику и рубрика для ревью. Каждый пункт проверяем. +Код-блоки, инлайн-код, шорткоды Hugo, URL и ключи front matter замаскированы пайплайном — их не переводят и не трогают ни при каких условиях; правила ниже касаются только прозы. + +## 1. Регистр и обращение + +- Обращение к читателю — на «вы» со строчной буквы (нейтрально-профессионально). «Вы» с прописной — ошибка. «Ты» — ошибка. +- Документация: безличный/императивный тон, без «мы», без эмоций. EN «We recommend using X» → ✗ «Мы рекомендуем использовать X» → ✓ «Рекомендуется использовать X». +- Блог и лендинг: «мы» допустимо, если это команда проекта; можно транскреация, но без восклицательных знаков, превосходных степеней и рекламных клише («революционный», «мощнейший»). +- EN «Please note that…» → ✓ «Обратите внимание:…» (никогда «пожалуйста»). EN «Let's create…» → ✓ «Создайте…». + +## 2. Заголовки + +- Инфинитивные заголовки, как в kubernetes.io/ru: EN «Installing a cluster» / «Install a cluster» → ✗ «Установка кластера» → ✓ «Установить кластер». +- Заголовки-существительные оставляем существительными: «Architecture» → «Архитектура», «Requirements» → «Требования». +- Заголовки-вопросы сохраняем: «Why Cozystack?» → «Почему Cozystack?». +- Английский Title Case → русская прописная только в первом слове и в именах собственных: «Configure Storage Backend» → ✗ «Настроить Хранилище Данных» → ✓ «Настроить бэкенд хранилища». +- Точка в конце заголовка не ставится. То же правило применяется к `title` и `description` во front matter. + +## 3. Терминология: правило решения (для терминов вне глоссария) + +Глоссарий (do_not_translate + preferred) инжектится отдельно и имеет приоритет. Для всего остального — по порядку: +1. Есть устоявшийся перевод в kubernetes.io/ru или CNCF Cloud Native Glossary → берём его. +2. Нет перевода, но термин читатель видит в CLI, UI, логах или манифестах → оставить латиницей, не склонять, добавить родовое слово: «в объекте PersistentVolumeClaim», «через контроллер Flux». +3. Есть точный русский аналог, не мешающий поиску → перевести, при первом упоминании на странице дать оригинал в скобках: «отказоустойчивость (fault tolerance)». Дальше — только русский вариант. +4. Транслитерация допустима только для уже укоренившихся форм: под, тенант, кластер, бэкенд, деплой. Не изобретать новых («кубелет», «инстанс» в доке — ✗, «экземпляр» — ✓). +5. Один термин — один перевод на всей странице и на всём сайте. Синонимическое разнообразие здесь — дефект, а не стиль. + +Объекты API Kubernetes: +- Как kind в манифесте — латиницей, с прописной, без склонения, с родовым словом: «создайте ресурс Deployment», «в поле spec объекта Secret». +- В обычной прозе, как нарицательное — по-русски и склоняется: pods → «поды», nodes → «узлы» (не «ноды»), secrets → «секреты», namespaces → «пространства имён» (но в командах и в описании флагов — namespace как есть). +- Устоявшиеся англицизмы инфраструктуры не переводить силой: «под» (в коде — pod), «узел» вместо «нода» в прозе, «развёртывание» предпочтительнее «деплоя». +- Не переводить и не склонять: kubectl, namespace (в командах), CLI-флаги, имена полей YAML, названия продуктов. + +## 4. Типографика и пунктуация + +- Кавычки — «ёлочки»; вложенные — „лапки“: «параметр „replicas“ в манифесте». ASCII-кавычки (" ') и “…” в прозе запрещены; внутри инлайн-кода остаются как есть. +- Тире — длинное (—), отбивается пробелами, слева — неразрывным. Дефис (-) — только внутри слов: «бэкап-политика», «SLA-требования». Диапазоны — короткое тире без пробелов: «3–5 узлов», «2024–2026». +- Пропуск глагола-связки оформляется тире: «Cozystack — платформа на базе Kubernetes». +- Многоточие — один символ «…», не «...». +- Буква ё обязательна: «развёртывание», «учётные данные», «объём», «затем». +- Неразрывный пробел: между числом и единицей («16 ГиБ», «3 узла»), внутри сокращений («и т. д.», «т. е.»), после коротких предлогов и союзов в начале синтагмы («в кластере», «с помощью»), перед длинным тире. +- Знаки препинания не отбиваются пробелом слева; двойных пробелов не бывает. +- Списки: если элементы продолжают вводную фразу — со строчной буквы и «;» на конце, последний — «.». Если элементы самостоятельные предложения — с прописной и точкой. Английский стиль (Title Case в пунктах, точки через один) не переносить. + +## 5. Числа, даты, единицы + +- Десятичный разделитель в прозе — запятая: «3.14» → «3,14»; «0.5 CPU» → «0,5 CPU». КРИТИЧНО: внутри кода, версий, тегов, значений YAML, размеров в командах разделитель НЕ меняется никогда: `v1.2.5`, `cpu: 0.5`, «версия 1.2.5» остаются с точкой. +- Разряды тысяч — неразрывный пробел, не запятая: «10,000 pods» → «10 000 подов». +- Проценты — слитно с числом: «50%». Диапазон — «10–15%». +- Единицы измерения в техническом контексте — латиницей и в оригинальной форме (GiB, Mi, vCPU, IOPS, ms), так как совпадают с тем, что читатель видит в манифестах. В блоге и маркетинге допустима кириллица (ГБ, ТБ) — но единообразно в пределах страницы. +- Даты в прозе — по-русски: «July 24, 2026» → «24 июля 2026 года». В командах, тегах, front matter и changelog формат ISO не трогать. +- Числа и существительные согласуются: «1 узел», «2 узла», «5 узлов». MT часто оставляет «5 узла» — проверять. + +## 6. Грамматические ловушки при переводе с английского + +- Артикли исчезают: «a cluster», «the node» → «кластер», «узел». Не превращать «a» в «один», «the» в «данный». +- Английский пассив → русский актив или безличная конструкция: «The volume is mounted by kubelet» → ✗ «Том монтируется kubelet-ом» → ✓ «Том монтирует kubelet» / ✓ «Том монтируется автоматически». +- Герундий в начале фразы → предлог или деепричастие, но не цепочка «используя»: «Using Flux, you can…» → ✗ «Используя Flux, вы можете…» → ✓ «С помощью Flux можно…». +- «You can / you need to» в документации → безлично: «You can override defaults» → ✓ «Значения по умолчанию можно переопределить». «Вы можете» оставляем только там, где важно противопоставление ролей (что может пользователь, а что — администратор). +- Цепочки существительных разворачиваются в родительный падеж, но не длиннее двух звеньев подряд: «Kubernetes cluster node pool configuration» → ✗ «конфигурация пула узлов кластера Kubernetes» → ✓ «настройка пулов узлов в кластере Kubernetes». Третий родительный подряд — разбить предлогом или прилагательным. +- Порядок слов: известное — в начало, новое — в конец. «A new tenant is created by the administrator» → ✓ «Новый тенант создаёт администратор» (если рема — кто) / ✓ «Администратор создаёт новый тенант» (если рема — что). +- Вид глагола: инструкция — совершенный вид («Создайте секрет», «Примените манифест»); описание повторяющегося поведения — несовершенный («Контроллер проверяет состояние каждые 30 секунд»). MT часто ставит несовершенный в шагах — ошибка. +- Согласование с аббревиатурами — по роду опорного слова: «API возвращает» (интерфейс), «ОС поддерживает» (система), «СУБД настроена» (система). Латинские аббревиатуры не склоняем: «через API», не «через APIшку». +- Английские имена собственные из списка do_not_translate не склоняются; если фраза становится неуклюжей — добавить родовое слово: ✗ «в Талосе» → ✓ «в Talos Linux», ✗ «Cozystack'ом» → ✓ «средствами Cozystack». +- Отрицание не терять: «must not be empty» → «не должно быть пустым», а не «должно быть пустым». + +## 7. Переводческие штампы и кальки (вычищать) + +- «В данной статье мы рассмотрим…» → «Здесь описано…» / «В статье описано…». «Данный» → «этот». +- «является» вместо тире: «Cozystack является платформой» → ✓ «Cozystack — платформа». +- «позволяет вам», «предоставляет возможность», «дает возможность» → «позволяет», «даёт», а лучше прямое действие: «Flux allows you to sync manifests» → ✓ «Flux синхронизирует манифесты». +- «просто», «легко», «всего лишь» как калька от «simply/easily» — удалять: «Simply run the command» → ✓ «Выполните команду». +- «поддерживается» в каждом абзаце (support): «X supports Y» → «X работает с Y» / «в X есть Y» / «X совместим с Y» — варьировать по смыслу. +- «для того чтобы» (от «in order to») → «чтобы»; «в случае, если» → «если»; «в настоящее время» → «сейчас»; «осуществлять настройку» → «настраивать». +- «Обратите внимание, что» — только там, где в оригинале Note/Warning. Цепочки «который… который…» — разбивать на предложения. +- Буквальные «пожалуйста», «наслаждайтесь», «мы рады сообщить» — не переносить. + +## 8. Ложные друзья + +actual → «фактический», не «актуальный» · consistent → «согласованный», «единообразный», не «консистентный» · control (control plane, access control) → «управление», не «контроль» · validate → «проверить», не «валидировать» (кроме валидации схемы API) · accurate → «точный» · dramatic(ally) → «резко», «значительно», не «драматически» · eventually → «в итоге», «в конечном счёте» · transparent → «незаметный для пользователя», редко «прозрачный» · native → «встроенный», «собственный» (но cloud native — не переводим) · performance → «производительность», не «перформанс» · capacity → «ёмкость», «мощность» · executive → «руководитель» · application → «приложение», но в бизнес-контексте может быть «заявка» · commit (git) → не переводим; commitment → «обязательство» · legacy → «унаследованный», «устаревший» · instance → «экземпляр» · provision → «выделить», «подготовить» · resolution → «разрешение» (DNS) или «решение проблемы» — различать · scale → «масштабировать», не «шкалировать» · security → «безопасность», не «секьюрити». + +## 9. Чеклист ревьюера: типовые сбои машинного перевода на русский + +- Заголовок не в инфинитиве или скопирован Title Case. +- «Вы» с прописной; смена регистра обращения внутри страницы. +- ASCII-кавычки или английские “…” в прозе; дефис вместо тире; «...» вместо «…»; пропущенная ё; двойные пробелы; отсутствие неразрывных пробелов в «16 ГиБ», «и т. д.». +- Десятичная точка в прозе — или, наоборот, запятая, вкравшаяся в версию/код/значение YAML; английский разделитель тысяч (10,000). +- Один и тот же термин переведён по-разному в разных абзацах; термин из глоссария проигнорирован. +- Переведено то, что переводить нельзя: имя kind, флаг, поле YAML, часть URL, текст внутри плейсхолдера §§…§§. +- Склонение или транслитерация латинских имён собственных («Кубернетесе», «Козистэк»). +- Рассогласование числительного и существительного, рода при аббревиатуре. +- Английская синтаксическая рамка: пассив, «вы можете», «используя», цепочки из трёх и более родительных падежей. +- Штампы из раздела 7 («является», «в данной статье», «позволяет вам», «просто»). +- Потерянное отрицание, потерянные условия («only if», «unless»), перепутанные «may/must/should». +- Незакрытая логика списка: пункты с разной пунктуацией и разным регистром первой буквы. +- Нарушена структура Markdown: съеден уровень заголовка, потеряна колонка таблицы, переведён URL внутри ссылки. diff --git a/hack/i18n/style-guides/zh-cn.md b/hack/i18n/style-guides/zh-cn.md new file mode 100644 index 000000000..fd7d71e1f --- /dev/null +++ b/hack/i18n/style-guides/zh-cn.md @@ -0,0 +1,95 @@ +- Simplified Chinese (zh-Hans), mainland conventions; follow kubernetes.io/zh-cn glossary. +- No spaces between Chinese characters; put a space between Chinese and Latin/number runs. +- Use full-width Chinese punctuation (，。：；""）for Chinese text; keep half-width for code/commands. +- Keep product/brand/CLI terms in Latin; translate prose. Common infra terms per k8s zh glossary (集群=cluster, 节点=node, 存储=storage, 工作负载=workload). + +### 1. Register and address + +- Docs: 书面语 — precise, impersonal, imperative for steps. Blog/landing: livelier, shorter sentences, rhetorical questions allowed; never 网络流行语 or exclamation spam. +- Address the reader as **你**, never 您. This is the kubernetes.io/zh-cn convention: 您 reads deferential/sales-y and is impossible to keep consistent across a large doc set. Better still, drop the pronoun. EN "You can install it with Helm" → ✗ 您可以使用 Helm 来安装它 → ✓ 使用 Helm 安装即可。 +- Mixing 你 and 您 on one page is a defect even if each sentence reads fine. +- First person plural only in signed blog posts. Docs: ✗ 我们建议启用备份 → ✓ 建议启用备份。 + +### 2. Headings + +- No trailing 。 in any heading. ？and ！are allowed when the source is a question/exclamation: "What is Cozystack?" → ✓ 什么是 Cozystack？ +- Concept/reference headings → noun phrase: "Storage Architecture" → ✓ 存储架构（✗ 存储是如何架构的）。 +- Task headings → 动宾 verb phrase, no 正在 / 如何进行 padding: "Installing Cozystack" → ✗ 正在安装 Cozystack、✗ 如何进行 Cozystack 的安装 → ✓ 安装 Cozystack。 +- Conventional renderings: "Getting Started" → 快速开始；"Prerequisites" → 前置条件；"Troubleshooting" → 故障排查；"See also" → 参见；"Next steps" → 后续步骤。 +- Do not add numbering the source lacks; keep heading levels unchanged. + +### 3. Terminology policy (terms NOT in the injected lists) + +Decision order: (1) kubernetes.io/zh-cn glossary → (2) CNCF Cloud Native Glossary 中文版 → (3) rules below. Never coin a new Chinese term for a concept the Chinese k8s community discusses in English. +- **API kinds, CRD kinds, field names, flags, CLIs stay Latin and uninflected**: Pod、Deployment、StatefulSet、DaemonSet、Secret、ConfigMap、Ingress、kubelet、kubectl。✗ 豆荚 / 入口 / 秘密 / 配置映射。 +- **The same word as an ordinary noun is translated** per k8s zh: node→节点、namespace→命名空间、volume→卷、label→标签、annotation→注解、container→容器、image→镜像、replica→副本、control plane→控制平面、service→服务。If the sentence means `kind: Node` / `kind: Service`, keep Node / Service in Latin. +- Generic concepts with stable renderings are translated: 高可用、可观测性、编排、调度、快照、备份、故障转移、隔离、租户。 +- Unknown/new term with no stable rendering: keep Latin, or use 中文（English）on **first mention per page**, Chinese only afterwards — ✓ 单节点控制平面（single-node control plane）。Never repeat the bracketed English on every occurrence. +- One page = one rendering per term. ✗ 租户 … 承租方 … 租客 in the same file. + +### 4. Punctuation + +- Full-width only in Chinese prose: ，。：；！？（）【】、 — ASCII `,` `.` `;` `:` in Chinese prose is always a defect. +- **顿号 、 separates coordinate items inside a sentence** (English has no equivalent — do not copy the source commas): "CPU, memory, and storage" → ✗ CPU，内存，和存储 → ✓ CPU、内存和存储。 +- Quotes: “ ” (inner ‘ ’) for quoted prose and UI strings; 《》 for titles of docs/books/specs: ✓ 详见《Cozystack 安装指南》。✗ "Cozystack 安装指南"。 +- Ellipsis is ……（two full-width chars）, dash is ——, never `...` or ` - `. Ranges: ✓ 3～5 个节点 / 3 到 5 个节点。 +- Half-width is correct and required inside code, inline code, versions (v1.5.0), paths, URLs, decimals (99.9), flags (`--dry-run`), and inside a fully English quoted string. +- No doubled punctuation: ✗ 部署完成。）→ ✓ 部署完成）。 Bullets and table cells: no 。 on fragments; 。 on full sentences, consistently within one list. +- Parentheses inside a Chinese sentence are full-width （） even when the content is Latin: ✓ 控制平面（control plane）。 + +### 5. Spacing (pangu) + +- One space between a Chinese run and an adjacent Latin/digit run: ✓ 使用 Helm 部署 3 个副本。 ✗ 使用Helm部署3个副本。 +- **No** space between Chinese/Latin and full-width punctuation: ✗ 安装 Cozystack ，然后… → ✓ 安装 Cozystack，然后…；✗ 存储 、 网络 → ✓ 存储、网络。 +- Keep the space around masked inline-code spans too: ✓ 运行 `kubectl get pods` 查看状态。 +- Never insert pangu spaces inside code, paths, URLs, or masked placeholders. +- Never use full-width Latin letters or digits (✗ Ｋｕｂｅｒｎｅｔｅｓ、✗ １０); no double spaces; no space before a line-final 。 + +### 6. Numbers, dates, units + +- Arabic numerals for all quantities, versions, dates, ports, sizes. Chinese numerals only in fixed expressions (一致、一次性、第三方). Procedures: ✓ 第 1 步 / 步骤 1。 +- 万/亿 are fine in blog prose for round numbers（✓ 10 万个 Pod）; in docs keep the source's exact figures and separators (1,000 QPS stays 1,000 QPS). Never rescale magnitudes silently. +- English "billion" = 10 亿, "trillion" = 万亿 — a classic MT slip; verify every large number against the source. +- Dates: ✓ 2026年7月24日（no spaces around 年月日）; ✗ 7月24日，2026。Times: ✓ 14:30。 +- Units: space between number and Latin unit — ✓ 16 GiB、100 ms、10 Gbit/s；no space for % or Chinese measure words — ✓ 99.9%、3 个节点。 + +### 7. Grammar traps translating from English + +- **量词 are mandatory and must be right**: "three nodes" → ✗ 3 节点 → ✓ 3 个节点。一台虚拟机、一块 GPU、一份清单、一条规则、一组副本、一次请求、一个 Pod、一套集群。 +- **Attributive chains before 的**: at most one 的 per noun phrase; move the rest into a following clause. "a highly available, self-healing Kubernetes control plane deployed on bare-metal nodes" → ✗ 一个部署在裸金属节点上的高可用的能够自我修复的 Kubernetes 控制平面 → ✓ 一个高可用的 Kubernetes 控制平面，部署在裸金属节点上，并可自我修复。 +- **Articles and plurals go to zero**: ✗ Pod们被调度 → ✓ Pod 会被调度；✗ 这些的集群 → ✓ 这些集群。 +- **被 is overused by MT.** Named agent → 由: ✗ 卷被 kubelet 挂载 → ✓ 卷由 kubelet 挂载。No agent → topic-comment: ✗ 备份被存储在 S3 中 → ✓ 备份存储在 S3 中。Reserve 被 for adverse/unexpected events. +- **Split long sentences**: one English sentence with 2+ subordinate clauses → 2–3 Chinese clauses ended by 。, ideally ≤ 40 汉字 each. Do not chain them with ；. +- **Topic first (主题-述题)**: "You can configure the number of replicas in values.yaml." → ✓ 副本数量在 `values.yaml` 中配置。 +- **Condition before result**: "Restart the Pod if the probe fails." → ✓ 如果探针失败，则重启 Pod。 +- 的/地/得: 快速的部署（attributive）、快速地部署（adverbial）、部署得很快（complement）— MT confuses these constantly. + +### 8. 翻译腔 patterns to eliminate + +- 进行/做出/实现 + nominalization: ✗ 对集群进行备份操作 → ✓ 备份集群；✗ 做出配置更改 → ✓ 修改配置。 +- 一个 for every English "a": ✗ 这是一个常见的模式 → ✓ 这是常见模式。 +- 们 on generic/inanimate nouns: ✗ 用户们、开发者们、节点们 → ✓ 用户、开发者、节点。 +- 关于/对于 padding: ✗ 关于更多关于 X 的信息，请参见 Y → ✓ 更多信息参见 Y；✗ 对 X 的支持 → ✓ 支持 X。 +- "simply / just / easily": ✗ 简单地运行以下命令 → ✓ 运行以下命令（或 只需运行以下命令）。 +- "you can" on every sentence: ✗ 你可以使用 Helm 来安装它 → ✓ 使用 Helm 安装。 +- 来 as filler infinitive: ✗ 使用 kubectl 来查看日志 → ✓ 使用 kubectl 查看日志。 +- "allows/enables you to": ✗ 允许你创建租户 → ✓ 支持创建租户。 +- "It is worth noting that…": ✗ 值得注意的是，… → ✓ 注意：… +- 请 only where the source is a genuine pointer/request（✓ 请参见）, not on every imperative. + +### 9. Reviewer checklist — Chinese-specific MT failure modes + +1. 繁体字或台港用词泄漏: ✗ 記憶體/軟體/網路/程式/預設/伺服器/映像檔/叢集 → ✓ 内存/软件/网络/程序/默认/服务器/镜像/集群；✗ 结点 → ✓ 节点。 +2. ASCII punctuation in Chinese prose; missing 顿号; `...` instead of ……; half-width () wrapping Chinese. +3. Missing or spurious pangu spaces; full-width Latin/digits; space before 。，、. +4. Over-translated identifiers (Pod/Ingress/Secret/kubectl rendered in Chinese) or under-translated prose (cluster/node left in English). +5. Term drift within a page; first-mention 中文（English）repeated on every occurrence. +6. 被-heavy passives; 的-chains with more than one 的; missing 量词; 的/地/得 confusion. +7. English word order in Chinese characters — read the sentence aloud; if it needs a second pass to parse, split it. +8. 你/您 mixing; register drift (marketing tone inside reference docs, or stiff 书面语 inside a blog post). +9. Number/magnitude errors, altered versions, flags, resource names, or YAML keys. +10. Untranslated leftovers, duplicated clauses, and **dropped negation**（未/不/无 lost reverses the meaning）; lost admonition labels — note→说明、warning→警告、caution→注意。 + +### 10. Structural (pipeline-enforced) + +Code blocks, inline code, Hugo shortcodes（`{{< >}}` / `{{% %}}`）, URLs, file paths, HTML comments and front-matter **keys** are masked as `§§NAME_N§§` placeholders: copy them through verbatim, never translate, reorder, or space-fix their contents. Translate link **text**, never the URL; keep heading levels, list markers, and table column counts identical to the source. diff --git a/hack/i18n/test_i18n.py b/hack/i18n/test_i18n.py new file mode 100644 index 000000000..5f0102b74 --- /dev/null +++ b/hack/i18n/test_i18n.py @@ -0,0 +1,1024 @@ +#!/usr/bin/env python3 +"""Tests for the pure functions behind the translation pipeline. + +No network, no model calls, no Hugo: everything here is deterministic. Run with +`python3 -m pytest hack/i18n/test_i18n.py` (or plain `python3 hack/i18n/test_i18n.py`). + +These cover the failure modes that are silent in production — a dropped code +fence, a reverted hero, a page that looks permanently stale, a reviewer verdict +read as a pass. Each one shipped a real bug during development. +""" + +from __future__ import annotations + +import os +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import lib +import translate + + +class TestProtectRestore(unittest.TestCase): + def test_roundtrip_is_lossless(self): + text = ( + "Intro `inline` text.\n\n" + "```yaml\nkey: value\n```\n\n" + "{{< figure src=\"a.png\" >}}\n\n" + "<!-- a comment -->\n" + ) + masked, store = lib.protect(text) + self.assertEqual(lib.restore(masked, store), text) + + def test_code_is_hidden_from_the_model(self): + masked, store = lib.protect("before\n```bash\nkubectl get pods\n```\nafter") + self.assertNotIn("kubectl", masked) + self.assertEqual(len(store), 1) + + def test_fence_wins_over_inline_code(self): + # A fence contains backticks; if INLINECODE ran first it would shred it. + masked, store = lib.protect("```\na `b` c\n```") + self.assertEqual(len(store), 1) + self.assertTrue(next(iter(store)).startswith("§§FENCE_")) + + def test_figure_caption_and_alt_are_exposed_for_translation(self): + # The bug a virtual reviewer caught: captions render under the image and + # were staying English because the whole shortcode was masked. + sc = '{{< figure src="a.webp" alt="Dashboard view" width="720" caption="The new dashboard." >}}' + masked, store = lib.protect(sc) + self.assertIn("Dashboard view", masked) # alt exposed + self.assertIn("The new dashboard.", masked) # caption exposed + self.assertNotIn("a.webp", masked) # src protected + self.assertNotIn("720", masked) # width protected + + def test_figure_structure_survives_translated_caption(self): + sc = '{{< figure src="a.webp" alt="A" width="720" caption="B" >}}' + masked, store = lib.protect(sc) + translated = masked.replace("A", "Я").replace("B", "Б") + self.assertEqual( + lib.restore(translated, store), + '{{< figure src="a.webp" alt="Я" width="720" caption="Б" >}}') + + def test_shortcode_without_visible_attrs_is_masked_wholesale(self): + masked, store = lib.protect('{{< youtube id="abc123" >}}') + self.assertNotIn("abc123", masked) + self.assertEqual(len(store), 1) + + +class TestHtmlProtection(unittest.TestCase): + """`.html` pages are mostly markup, and Markdown embeds it freely. An + unprotected tag reaches the model, which is free to translate an attribute + or drop a closing tag — damage Hugo renders without complaint.""" + + def test_raw_tags_are_hidden_from_the_model(self): + masked, _ = lib.protect('<div class="hero-gitops">text</div>') + self.assertNotIn("hero-gitops", masked) + self.assertNotIn("<div", masked) + self.assertIn("text", masked) # prose still reaches the model + + def test_html_page_yields_placeholders_to_validate(self): + # With no placeholders at all there is nothing for the payload check to + # fail closed on, which is how markup damage used to ship silently. + _, store = lib.protect('<a href="/x" target="_blank">Read</a><br />') + self.assertTrue(store) + + def test_visible_attributes_still_translate(self): + # Masking a tag wholesale would freeze alt/title text in English. + masked, _ = lib.protect('<img src="a.png" alt="A running cluster" />') + self.assertIn("A running cluster", masked) + self.assertNotIn("a.png", masked) + + def test_prose_comparison_is_not_swallowed(self): + text = "when x <y it holds" + masked, store = lib.protect(text) + # `<y` is a prose comparison, not a tag: it must reach the model unmasked. + # A round-trip alone cannot catch an over-greedy tag pattern (any mask + # round-trips), so assert the comparison stays visible and nothing was masked. + self.assertIn("<y", masked) + self.assertEqual(store, {}) + self.assertEqual(lib.restore(masked, store), text) + + def test_tag_inside_inline_code_is_protected_once(self): + text = "use `<div>` here" + masked, store = lib.protect(text) + # The inline code is masked as one unit; the <div> inside must NOT be masked + # again as a separate tag token (which a round-trip would not reveal). + self.assertEqual(len(store), 1) + self.assertTrue(next(iter(store)).startswith("§§INLINECODE_")) + self.assertEqual(lib.restore(masked, store), text) + + +class TestNestedPlaceholders(unittest.TestCase): + """Protected spans nest, and restoring in insertion order used to reveal an + inner token only after its own turn had passed — publishing a literal + `§§SC_0§§` into the page.""" + + def test_shortcode_inside_inline_code_survives(self): + text = '| `{{< version-pin "cozystack_version" >}}` | pin it |' + masked, store = lib.protect(text) + restored = lib.restore(masked, store) + self.assertEqual(restored, text) + self.assertNotIn("§§", restored) + + +class TestGlobMatching(unittest.TestCase): + def test_star_md_matches_basename_at_any_depth(self): + # Intended breadth: config.yaml's `*.md` means "a markdown page anywhere". + # Scope is narrowed afterwards by _docs_out_of_scope / _blog_too_old. + self.assertTrue(lib._matches_any("page.md", ["*.md"])) + self.assertTrue(lib._matches_any("docs/v1.5/deep/page.md", ["*.md"])) + + def test_star_md_does_not_match_other_extensions(self): + self.assertFalse(lib._matches_any("page.html", ["*.md"])) + + def test_single_star_does_not_swallow_a_subtree(self): + # The fnmatch trap: its `*` matches `/` too, so `docs/*` would match at + # any depth. Path semantics keep it to one level. + self.assertTrue(lib._matches_any("docs/x.md", ["docs/*"])) + self.assertFalse(lib._matches_any("docs/v1.5/x.md", ["docs/*"])) + + def test_subtree_glob(self): + self.assertTrue(lib._matches_any("docs/v1.5/x.md", ["docs/**"])) + self.assertFalse(lib._matches_any("blog/x.md", ["docs/**"])) + + def test_leading_doublestar(self): + self.assertTrue(lib._matches_any("docs/_include/x.md", ["**/_include/**"])) + + +class TestScope(unittest.TestCase): + def test_only_latest_docs_version(self): + self.assertFalse(lib._docs_out_of_scope("docs/v1.5/intro.md", "v1.5")) + self.assertTrue(lib._docs_out_of_scope("docs/v1.4/intro.md", "v1.5")) + self.assertTrue(lib._docs_out_of_scope("docs/next/intro.md", "v1.5")) + + def test_docs_version_picker_landing_in_scope(self): + # docs/_index.md is the version-picker, not a versioned page — it must + # stay in scope so the pipeline keeps its translations fresh. + self.assertFalse(lib._docs_out_of_scope("docs/_index.md", "v1.5")) + # A versioned landing still narrows to the latest version. + self.assertFalse(lib._docs_out_of_scope("docs/v1.5/_index.md", "v1.5")) + self.assertTrue(lib._docs_out_of_scope("docs/v1.4/_index.md", "v1.5")) + + def test_non_docs_paths_are_untouched(self): + self.assertFalse(lib._docs_out_of_scope("blog/2026-06-01-post.md", "v1.5")) + + def test_blog_cutoff(self): + self.assertTrue(lib._blog_too_old("blog/2026-01-01-old.md", "2026-05-17")) + self.assertFalse(lib._blog_too_old("blog/2026-06-01-new.md", "2026-05-17")) + + def test_blog_cutoff_disabled_by_empty_string(self): + self.assertFalse(lib._blog_too_old("blog/2020-01-01-ancient.md", "")) + + def test_undated_blog_paths_are_kept(self): + self.assertFalse(lib._blog_too_old("blog/_index.md", "2026-05-17")) + + def test_blog_rolling_window(self): + import datetime + recent = (datetime.date.today() - datetime.timedelta(days=10)).isoformat() + ancient = (datetime.date.today() - datetime.timedelta(days=90)).isoformat() + self.assertFalse(lib._blog_too_old(f"blog/{recent}-post.md", "60d")) + self.assertTrue(lib._blog_too_old(f"blog/{ancient}-post.md", "60d")) + # An undated path is kept regardless of the window form. + self.assertFalse(lib._blog_too_old("blog/_index.md", "60d")) + + +class TestTranscreatedPages(unittest.TestCase): + """A page marked `l10n: transcreate` was written by a person. Re-translating + it replaces that work with literal machine output AND overwrites the marker + with `mt`, so the loss is silent and unrepeatable.""" + + CFG = {"content_dir": "content", "source_lang": "en", + "languages": [{"code": "ru"}, {"code": "de"}], + "translate_globs": ["*.md", "*.html"]} + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self._old_root = lib.REPO_ROOT + lib.REPO_ROOT = self._tmp.name + self.addCleanup(lambda: setattr(lib, "REPO_ROOT", self._old_root)) + self._write("hugo.yaml", "params:\n latest_version_id: v1.6\n") + self._write("content/en/_index.html", "<h1>Cozystack</h1>\n") + + def _write(self, rel, text): + path = os.path.join(self._tmp.name, rel) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as fh: + fh.write(text) + return path + + STALE_MT = '---\nl10n: mt\nsource_digest: "sha256:stale"\n---\nbody\n' + STALE_HUMAN = '---\nl10n: transcreate\nsource_digest: "sha256:stale"\n---\nbody\n' + + def test_machine_page_with_moved_source_is_retranslated(self): + self._write("content/ru/_index.html", self.STALE_MT) + rels = [(i.lang, i.rel) for i in lib.build_worklist(self.CFG, only_lang="ru")] + self.assertIn(("ru", "_index.html"), rels) + + def test_hand_written_page_is_left_alone(self): + self._write("content/ru/_index.html", self.STALE_HUMAN) + rels = [(i.lang, i.rel) for i in lib.build_worklist(self.CFG, only_lang="ru")] + self.assertNotIn(("ru", "_index.html"), rels) + + def test_hand_written_page_is_reported_instead(self): + self._write("content/ru/_index.html", self.STALE_HUMAN) + self.assertIn(("ru", "_index.html"), + lib.find_transcreated(self.CFG, only_lang="ru")) + + def test_missing_translation_is_still_created(self): + # Nothing to protect when the page does not exist yet. + rels = [(i.lang, i.rel) for i in lib.build_worklist(self.CFG, only_lang="de")] + self.assertIn(("de", "_index.html"), rels) + + def test_up_to_date_hand_written_page_is_not_reported(self): + digest = lib.sha256_file(os.path.join(self._tmp.name, "content/en/_index.html")) + self._write("content/ru/_index.html", + f'---\nl10n: transcreate\nsource_digest: "sha256:{digest}"\n---\nx\n') + self.assertEqual(lib.find_transcreated(self.CFG, only_lang="ru"), []) + + def test_l10n_mode_reads_a_single_quoted_value(self): + # `l10n: 'transcreate'` is the same YAML value as the bare form. A regex that + # accepted only bare/double-quoted forms returned None here, sending a + # hand-authored page back into the worklist to be silently re-translated. + p = self._write("content/ru/q.md", "---\nl10n: 'transcreate'\n---\nbody\n") + self.assertEqual(lib.l10n_mode(p), "transcreate") + + +class TestOrphanTranslations(unittest.TestCase): + """A deleted English page must take its pipeline-managed translations with + it — and ONLY those: a hand-authored locale-only page has no source_digest + and must never be swept up.""" + + CFG = {"content_dir": "content", "source_lang": "en", + "languages": [{"code": "ru"}, {"code": "de"}]} + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self._old_root = lib.REPO_ROOT + lib.REPO_ROOT = self._tmp.name + self.addCleanup(lambda: setattr(lib, "REPO_ROOT", self._old_root)) + # Orphan detection now consults the latest docs version to sweep + # superseded translations, so the fixture needs a hugo.yaml to read it + # from — the same single source of truth the pipeline uses. + self._write("hugo.yaml", "params:\n latest_version_id: v1.6\n") + + def _write(self, rel, text): + path = os.path.join(self._tmp.name, rel) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as fh: + fh.write(text) + return path + + STAMPED = '---\ntitle: x\nsource_digest: "sha256:abc"\n---\nbody\n' + UNSTAMPED = "---\ntitle: x\n---\nbody\n" + + def test_translation_of_deleted_source_is_an_orphan(self): + orphan = self._write("content/ru/docs/gone.md", self.STAMPED) + self.assertEqual(lib.find_orphan_translations(self.CFG), [orphan]) + + def test_translation_with_live_source_is_kept(self): + self._write("content/en/docs/page.md", "---\ntitle: x\n---\nhello\n") + self._write("content/ru/docs/page.md", self.STAMPED) + self.assertEqual(lib.find_orphan_translations(self.CFG), []) + + def test_hand_authored_page_without_stamp_is_never_touched(self): + self._write("content/ru/docs/handmade.md", self.UNSTAMPED) + self.assertEqual(lib.find_orphan_translations(self.CFG), []) + + def test_only_lang_filter(self): + ru = self._write("content/ru/docs/gone.md", self.STAMPED) + de = self._write("content/de/docs/gone.md", self.STAMPED) + self.assertEqual(lib.find_orphan_translations(self.CFG, only_lang="ru"), [ru]) + self.assertEqual(sorted(lib.find_orphan_translations(self.CFG)), sorted([ru, de])) + + def test_superseded_docs_version_is_an_orphan_even_with_live_source(self): + # An old docs version has a live English source but is noindex and never + # refreshed; its stamped translation would rot forever, so it must be + # swept. The latest version, with the same live source, is kept. + self._write("content/en/docs/v1.4/x.md", "---\ntitle: x\n---\nhi\n") + self._write("content/en/docs/v1.6/x.md", "---\ntitle: x\n---\nhi\n") + old = self._write("content/ru/docs/v1.4/x.md", self.STAMPED) + self._write("content/ru/docs/v1.6/x.md", self.STAMPED) + self.assertEqual(lib.find_orphan_translations(self.CFG, only_lang="ru"), [old]) + + def test_superseded_orphans_above_floor_are_still_removed(self): + # A version bump orphans a whole retired docs version at once. Those + # superseded orphans must NOT count toward the moved-tree floor, or one + # release freezes all orphan cleanup — including a genuinely deleted page. + # Six superseded (> the floor if lumped) plus one deleted-source: the floor + # sees only the one deleted, does not trip, and all seven are removed. + for i in range(6): + self._write(f"content/en/docs/v1.4/p{i}.md", "---\ntitle: x\n---\nhi\n") + self._write(f"content/en/docs/v1.6/p{i}.md", "---\ntitle: x\n---\nhi\n") + self._write(f"content/ru/docs/v1.4/p{i}.md", self.STAMPED) + deleted = self._write("content/ru/docs/gone.md", self.STAMPED) + orig_wl, orig_tc, orig_argv = lib.build_worklist, lib.find_transcreated, sys.argv + try: + lib.build_worklist = lambda *a, **k: [] + lib.find_transcreated = lambda *a, **k: [] + sys.argv = ["translate.py", "--lang", "ru"] + translate.main() + finally: + lib.build_worklist, lib.find_transcreated, sys.argv = orig_wl, orig_tc, orig_argv + for i in range(6): + self.assertFalse( + os.path.exists(os.path.join(self._tmp.name, f"content/ru/docs/v1.4/p{i}.md")), + "superseded orphan was frozen by the floor") + self.assertFalse(os.path.exists(deleted), "deleted-source orphan was not removed") + + def test_version_agnostic_docs_landing_is_never_an_orphan(self): + # docs/_index.md sits directly under docs/ (no version) and stays in + # scope; a live source keeps its translation. + self._write("content/en/docs/_index.md", "---\ntitle: x\n---\nhi\n") + self._write("content/ru/docs/_index.md", self.STAMPED) + self.assertEqual(lib.find_orphan_translations(self.CFG, only_lang="ru"), []) + + +class TestRecordedDigest(unittest.TestCase): + def _write(self, text): + fh = tempfile.NamedTemporaryFile("w", suffix=".md", delete=False, encoding="utf-8") + fh.write(text) + fh.close() + self.addCleanup(os.unlink, fh.name) + return fh.name + + def test_reads_digest(self): + p = self._write('---\ntitle: x\nsource_digest: "sha256:abc123"\n---\nbody\n') + self.assertEqual(lib.recorded_digest(p), "abc123") + + def test_digest_after_a_long_front_matter(self): + # A fixed-size read would miss this, the page would look permanently + # stale, and it would be re-translated (and re-billed) every single day. + filler = "\n".join(f"k{i}: {'v' * 80}" for i in range(200)) + p = self._write(f'---\n{filler}\nsource_digest: "sha256:deadbeef"\n---\nbody\n') + self.assertEqual(lib.recorded_digest(p), "deadbeef") + + def test_missing_file_and_missing_digest(self): + self.assertIsNone(lib.recorded_digest("/nonexistent/page.md")) + self.assertIsNone(lib.recorded_digest(self._write("---\ntitle: x\n---\nbody\n"))) + + def test_body_mentioning_source_digest_is_not_read(self): + p = self._write('---\ntitle: x\n---\nsource_digest: "sha256:fake"\n') + self.assertIsNone(lib.recorded_digest(p)) + + +class TestNestedFrontMatter(unittest.TestCase): + FM = { + "title": "T", + "slug": "keep-me", + "taglines": ["A", "B"], + "benefits": [{"title": "BT", "icon": "fas fa-code", "description": "BD"}], + "seo": {"title": "ST", "keywords": ["K1"]}, + } + + def test_extracts_nested_user_visible_copy(self): + got = lib.extract_translatable(self.FM) + self.assertEqual( + set(got), + {"title", "taglines[0]", "taglines[1]", "benefits[0].title", + "benefits[0].description", "seo.title", "seo.keywords[0]"}, + ) + + def test_structural_keys_are_never_extracted(self): + got = lib.extract_translatable(self.FM) + self.assertNotIn("slug", got) + self.assertNotIn("benefits[0].icon", got) + + def test_apply_translates_in_place_by_path(self): + out = lib.apply_translations(self.FM, { + "taglines[1]": "Б", "benefits[0].description": "БД", "seo.keywords[0]": "К1"}) + self.assertEqual(out["taglines"], ["A", "Б"]) + self.assertEqual(out["benefits"][0]["description"], "БД") + self.assertEqual(out["benefits"][0]["icon"], "fas fa-code") # untouched + self.assertEqual(out["seo"]["keywords"], ["К1"]) + self.assertEqual(out["slug"], "keep-me") + + def test_apply_does_not_mutate_the_source(self): + lib.apply_translations(self.FM, {"taglines[0]": "MUT"}) + self.assertEqual(self.FM["taglines"][0], "A") + + def test_stale_path_is_skipped_not_fatal(self): + out = lib.apply_translations(self.FM, {"benefits[9].title": "ghost"}) + self.assertEqual(len(out["benefits"]), 1) + + def test_target_only_keys_survive(self): + # A human added `seo:` and `l10n:` to the localized page; regenerating + # from the English front matter must not delete their work. + out = lib.merge_target_only_keys( + {"title": "T"}, {"title": "old", "l10n": "transcreate", "seo": {"title": "S"}}) + self.assertEqual(out["title"], "T") # translation wins + self.assertEqual(out["l10n"], "transcreate") # human addition kept + self.assertEqual(out["seo"], {"title": "S"}) + + def test_merge_tolerates_no_existing_page(self): + self.assertEqual(lib.merge_target_only_keys({"title": "T"}, None), {"title": "T"}) + + +class TestGlossary(unittest.TestCase): + def setUp(self): + self.g = lib.load_glossary() + self.cfg = lib.load_config() + + def test_every_enabled_language_has_every_preferred_term(self): + # zh-cn and hi shipped with zero preferred terms while being live, so + # their translations had nothing keeping terminology consistent. + pref = self.g["preferred"] + for lang in (l["code"] for l in self.cfg["languages"]): + missing = sorted(t for t, p in pref.items() if lang not in p) + self.assertEqual(missing, [], f"{lang} has no preferred rendering for: {missing}") + + def test_terms_in_both_lists_differ_only_by_case(self): + # `Tenant` (API kind, verbatim) vs `tenant` (ordinary noun, translated) + # is deliberate. Any OTHER overlap is a genuine contradiction: the + # translator would be told both to keep and to translate the same term. + dnt = {t.lower(): t for t in self.g["do_not_translate"]} + for term in self.g["preferred"]: + clash = dnt.get(term.lower()) + if clash is not None: + self.assertNotEqual( + clash, term, + f"{term!r} is in do_not_translate AND preferred with identical case") + + def test_do_not_translate_has_no_duplicates(self): + dnt = self.g["do_not_translate"] + self.assertEqual(len(dnt), len(set(dnt))) + + +class TestRealContentScope(unittest.TestCase): + """Runs against the actual content tree, so drift in the site shows up here.""" + + def setUp(self): + self.cfg = lib.load_config() + self.files = lib.iter_source_files(self.cfg) + + def test_scope_is_not_empty(self): + self.assertGreater(len(self.files), 50) + + def test_no_out_of_scope_paths_leak_in(self): + latest = lib.latest_docs_version(self.cfg) + for rel in self.files: + self.assertFalse(rel.startswith("oss-health/"), rel) + self.assertNotIn("/_include/", f"/{rel}") + # Versioned docs pages (docs/<ver>/...) must be the latest version; + # the version-picker landing (docs/_index.md) sits directly under + # docs/ and is version-agnostic, so it is allowed. + if rel.startswith("docs/") and rel.count("/") >= 2: + self.assertTrue(rel.startswith(f"docs/{latest}/"), f"stale docs version: {rel}") + + def _declared(self): + import yaml as _yaml + hugo = _yaml.safe_load(open(os.path.join(lib.REPO_ROOT, "hugo.yaml"), encoding="utf-8")) + return set(hugo["languages"]) - {self.cfg["source_lang"]} + + def test_every_served_language_is_translated(self): + # The invariant is one-directional. Translated-but-not-declared is the + # normal way a language starts; declared-but-not-translated means the + # site serves a language nothing keeps up to date. + configured = {l["code"] for l in self.cfg["languages"]} + orphaned = self._declared() - configured + self.assertEqual(orphaned, set(), + f"hugo.yaml serves {sorted(orphaned)} but hack/i18n/config.yaml " + f"does not translate them — they will never be created or refreshed") + + def test_every_served_language_has_content(self): + # Declaring a language with an empty content tree does not build + # nothing: Hugo still emits /<code>/, /<code>/tags/, /<code>/categories/, + # /<code>/topics/ and /<code>/404.html, all `index, follow`. + for code in self._declared(): + d = os.path.join(lib.REPO_ROOT, self.cfg["content_dir"], code) + self.assertTrue( + os.path.isdir(d) and os.listdir(d), + f"hugo.yaml declares '{code}' but content/{code}/ is empty or missing — " + f"that publishes empty indexable pages. Land the content first.") + + def test_no_unknown_user_visible_frontmatter_keys(self): + # Guards against a new user-visible field (like `lede` was) being added + # to content and silently shipping in English on every localized page. + known_visible = set(lib.TRANSLATABLE_FRONTMATTER_KEYS) | set(lib.TRANSLATABLE_LIST_KEYS) + # Structural or deliberately-untranslated keys. + allowed = { + "weight", "aliases", "date", "slug", "images", "author", "draft", "type", + "layout", "menu", "cascade", "taxonomyCloud", "source_digest", + "translation_status", "translation_review", "l10n", + # structural page keys (permalink override, body CSS class) + "url", "body_class", + # taxonomy terms: translating them would fork the taxonomy + "article_types", "topics", + # containers whose translatable children are reached by recursion + "benefits", "features", "use_cases", + } + seen = set() + for rel in self.files: + fm, _, _ = lib.split_frontmatter( + open(lib.source_path(self.cfg, rel), encoding="utf-8").read()) + if isinstance(fm, dict): + seen |= set(fm) + unknown = seen - known_visible - allowed + self.assertEqual(unknown, set(), + f"unrecognized front-matter key(s) {sorted(unknown)}: decide whether " + f"they are user-visible (add to lib.TRANSLATABLE_* ) or structural " + f"(add to `allowed` here)") + + +class TestVerdictParsing(unittest.TestCase): + def test_pass_and_revise(self): + self.assertEqual(translate._parse_verdict('{"verdict": "pass"}', "r")["verdict"], "pass") + self.assertEqual( + translate._parse_verdict('{"verdict": "revise", "findings": []}', "r")["verdict"], + "revise") + + def test_json_after_prose(self): + raw = 'Sure! Here is my review.\n\n{"verdict": "pass", "findings": []}' + self.assertEqual(translate._parse_verdict(raw, "r")["verdict"], "pass") + + def test_unparseable_fails_closed(self): + # The critical one: a refusal or prose reply must never read as a pass, + # or a broken prompt waves every page through and the gate is a no-op. + for raw in ("I cannot help with that.", "", "{truncated", '{"verdict": "maybe"}'): + v = translate._parse_verdict(raw, "editor") + self.assertEqual(v["verdict"], "revise", f"failed open on: {raw!r}") + self.assertTrue(v["findings"]) + + +class TestPayloadProtocol(unittest.TestCase): + def test_parses_json_frontmatter_and_body(self): + _, store = lib.protect("x") + fm, body = translate._split_payload_response( + '===FRONTMATTER===\n{"title": "Заголовок"}\n===BODY===\nтело', {}, {"title"}) + self.assertEqual(fm, {"title": "Заголовок"}) + self.assertEqual(body, "тело") + + def test_preamble_without_marker_is_rejected(self): + # Otherwise "Here's the translation:" gets written into the page. + with self.assertRaises(translate.ProtocolError): + translate._split_payload_response("Here's the translation!", {}, set()) + + def test_missing_frontmatter_key_is_rejected(self): + # A page whose body is translated but whose hero silently stayed English + # is worse than a retry tomorrow. + with self.assertRaises(translate.ProtocolError): + translate._split_payload_response( + '===FRONTMATTER===\n{"title": "T"}\n===BODY===\nb', {}, {"title", "taglines[0]"}) + + def test_dropped_placeholder_is_rejected(self): + masked, store = lib.protect("a\n```\ncode\n```\nb") + with self.assertRaises(translate.ProtocolError) as cm: + translate._split_payload_response( + '===FRONTMATTER===\n{}\n===BODY===\ntranslated, fence gone', store, set()) + self.assertIn("lost by the model", str(cm.exception)) + + def test_duplicated_placeholder_is_rejected(self): + masked, store = lib.protect("a\n```\ncode\n```\nb") + tok = next(iter(store)) + with self.assertRaises(translate.ProtocolError) as cm: + translate._split_payload_response( + f'===FRONTMATTER===\n{{}}\n===BODY===\n{tok}\n{tok}', store, set()) + self.assertIn("duplicated by the model", str(cm.exception)) + + def test_placeholders_are_restored(self): + masked, store = lib.protect("a\n```\ncode\n```\nb") + tok = next(iter(store)) + _, body = translate._split_payload_response( + f'===FRONTMATTER===\n{{}}\n===BODY===\nпереведено\n{tok}', store, set()) + self.assertIn("```\ncode\n```", body) + + def test_nested_placeholder_validates_only_top_level(self): + # A shortcode nested inside inline code leaves only the OUTER token in the + # masked body; the token-completeness check must consider only that + # top-level token, not the inner one that never appears in the body — else + # the page is rejected on every run and can never be translated. + masked, store = lib.protect('a `{{< version-pin "cozystack_version" >}}` b') + self.assertEqual(len(store), 2) # inner SC nested inside the outer INLINECODE + _, body = translate._split_payload_response( + f'===FRONTMATTER===\n{{}}\n===BODY===\ntranslated {masked}', store, set()) + self.assertIn('{{< version-pin "cozystack_version" >}}', body) + + def test_inner_token_the_model_never_saw_is_rejected(self): + # The mirror of the case above: the model may OMIT an inner token, but + # it must never EMIT one. A reply carrying the legit outer token plus a + # hallucinated inner token would otherwise restore the nested content + # twice — silent duplication the residual check can no longer catch. + masked, store = lib.protect("Use `{{< param version >}}` here.") + outer = next(t for t in store if t in masked) + inner = next(t for t in store if t not in masked) + with self.assertRaises(translate.ProtocolError) as cm: + translate._split_payload_response( + f'===FRONTMATTER===\n{{}}\n===BODY===\n{outer} и {inner}', + store, set()) + self.assertIn("injected by the model", str(cm.exception)) + + +class TestFindingsReport(unittest.TestCase): + def test_report_names_the_page_and_every_finding(self): + md = translate._format_report([{ + "lang": "ru", "rel": "docs/v1.5/intro.md", + "findings": [ + {"severity": "major", "from": "cozystack-maintainer", + "issue": "'node pool' rendered as 'пул узлов' but glossary says 'узел'"}, + {"severity": "minor", "from": "technical-editor", "issue": "register drifts formal"}, + ]}]) + self.assertIn("ru: docs/v1.5/intro.md", md) + self.assertIn("cozystack-maintainer", md) + self.assertIn("пул узлов", md) + self.assertIn("register drifts formal", md) + + def test_report_survives_a_malformed_finding(self): + # Findings come from a model; a finding missing `issue`/`severity` must + # still reach the maintainer rather than crash the run after the work. + md = translate._format_report([{"lang": "de", "rel": "x.md", "findings": [{}]}]) + self.assertIn("de: x.md", md) + + +class TestRateLimitMarkers(unittest.TestCase): + def test_real_limits_are_recognized(self): + for msg in ("rate_limit_error: too many", "You have hit your usage limit", + "credit balance is too low"): + self.assertTrue(any(m in msg.lower() for m in translate._RATE_LIMIT_MARKERS), msg) + + def test_transient_errors_are_not_treated_as_the_daily_limit(self): + # "overloaded" is a transient 529: swallowing it as the daily limit would + # end the run with exit 0, and a skipped day would look like a good one. + for msg in ("Error: overloaded_error", "connection reset", "500 internal"): + self.assertFalse(any(m in msg.lower() for m in translate._RATE_LIMIT_MARKERS), msg) + + +class TestDerefShortcodes(unittest.TestCase): + def test_absolute_ref_becomes_plain_link(self): + # The exact break shipped in the sample pages: a ref to a sibling not yet + # translated into this language hard-fails the Hugo build. + t = '[x]({{% ref "/docs/v1.5/getting-started/install-talos" %}})' + self.assertEqual( + lib.deref_shortcodes(t, "docs/v1.5/getting-started/install-kubernetes.md"), + "[x](/docs/v1.5/getting-started/install-talos/)") + + def test_angle_relref_and_fragment_are_preserved(self): + t = '{{< relref "/docs/v1.5/x#section" >}}' + self.assertEqual(lib.deref_shortcodes(t, "docs/v1.5/p.md"), "/docs/v1.5/x/#section") + + def test_relative_ref_resolves_against_page_dir(self): + t = '[y]({{% ref "install-cozystack" %}})' + self.assertEqual( + lib.deref_shortcodes(t, "docs/v1.5/getting-started/install-kubernetes.md"), + "[y](/docs/v1.5/getting-started/install-cozystack/)") + + def test_md_suffix_is_stripped(self): + self.assertEqual( + lib.deref_shortcodes('{{% ref "/docs/v1.5/x.md" %}}', "docs/v1.5/p.md"), + "/docs/v1.5/x/") + + def test_named_parameter_ref_is_rewritten(self): + t = '[a]({{% ref path="/docs/v1.5/x" %}})' + self.assertEqual(lib.deref_shortcodes(t, "docs/p.md"), "[a](/docs/v1.5/x/)") + t2 = '{{< relref path="/docs/v1.5/y#s" lang="de" >}}' + self.assertEqual(lib.deref_shortcodes(t2, "docs/p.md"), "/docs/v1.5/y/#s") + + def test_anchor_only_ref_targets_the_current_page(self): + t = 'See [this]({{% ref "#tenant-system" %}}).' + self.assertEqual( + lib.deref_shortcodes(t, "docs/v1.5/guides/concepts.md"), + "See [this](/docs/v1.5/guides/concepts/#tenant-system).") + + def test_text_without_shortcodes_is_untouched(self): + t = "no shortcodes here, just [a link](/docs/x/) and `code`." + self.assertEqual(lib.deref_shortcodes(t, "docs/p.md"), t) + + def test_has_ref_shortcode_detects_every_form(self): + self.assertTrue(lib.has_ref_shortcode('x {{% ref "/a" %}} y')) + self.assertTrue(lib.has_ref_shortcode('{{< relref path="/a" >}}')) + self.assertFalse(lib.has_ref_shortcode("plain [a](/a/) and `code`")) + + def test_unsupported_ref_shape_survives_and_is_detectable(self): + # No positional target, no path= — left intact so the fail-closed guard fires. + t = '{{% ref foo="bar" %}}' + self.assertTrue(lib.has_ref_shortcode(lib.deref_shortcodes(t, "docs/p.md"))) + + def test_backtick_raw_string_target_is_rewritten(self): + # Hugo accepts backtick raw-string args; strip them, don't leave a broken link. + t = "{{< ref `/docs/v1.5/x` >}}" + self.assertEqual(lib.deref_shortcodes(t, "docs/p.md"), "/docs/v1.5/x/") + + def test_section_index_target_maps_to_parent(self): + # `_index.md`/`index.md` are served at the parent URL, not `.../_index/`. + self.assertEqual( + lib.deref_shortcodes('{{% ref "/docs/v1.5/section/_index.md" %}}', "docs/p.md"), + "/docs/v1.5/section/") + self.assertEqual( + lib.deref_shortcodes('{{% ref "/docs/v1.5/section/index" %}}', "docs/p.md"), + "/docs/v1.5/section/") + + def test_malformed_target_fails_closed(self): + # A shape whose target can't be cleanly extracted must survive detection. + self.assertTrue(lib.has_ref_shortcode( + lib.deref_shortcodes('{{% ref bogus="x" other=1 %}}', "docs/p.md"))) + + +class TestRunStatus(unittest.TestCase): + def test_clean_run_produces_no_status(self): + self.assertEqual(translate._format_run_status(False, "", 5, [], 3), "") + + def test_early_stop_is_recorded(self): + md = translate._format_run_status(True, "usage limit", 7, [], 3) + self.assertIn("Run status", md) + self.assertIn("usage limit", md) + self.assertIn("7 page(s)", md) + + def test_skipped_pages_are_named(self): + md = translate._format_run_status( + False, "", 4, + [{"lang": "ru", "rel": "docs/v1.5/intro.md", "error": "no ===BODY==="}], 3) + self.assertIn("ru: docs/v1.5/intro.md", md) + self.assertIn("no ===BODY===", md) + + def test_skipped_survives_a_missing_error_field(self): + md = translate._format_run_status(False, "", 0, [{"lang": "de", "rel": "x.md"}], 3) + self.assertIn("de: x.md", md) + + def test_time_budget_stop_names_its_cause(self): + md = translate._format_run_status(True, "", 12, [], 3, + cause="the run's 240-minute time budget") + self.assertIn("240-minute time budget", md) + self.assertNotIn("usage limit", md) + + +class TestTimeBudget(unittest.TestCase): + def test_no_budget_never_stops(self): + self.assertFalse(translate._budget_exhausted(0, None, 10 ** 9)) + self.assertFalse(translate._budget_exhausted(0, 0, 10 ** 9)) + + def test_stops_once_the_budget_is_spent(self): + self.assertFalse(translate._budget_exhausted(100.0, 1, 159.9)) + self.assertTrue(translate._budget_exhausted(100.0, 1, 160.0)) + + +class TestOrphanFloor(unittest.TestCase): + def test_floor_value_is_pinned(self): + self.assertEqual(translate.ORPHAN_PAGE_FLOOR, 5) + + def test_distinct_pages_dedup_across_languages(self): + root = "/repo/content" + orphans = ["/repo/content/de/docs/x.md", "/repo/content/ru/docs/x.md", + "/repo/content/de/docs/y.md"] + self.assertEqual(translate._distinct_orphan_pages(orphans, root), + {"docs/x.md", "docs/y.md"}) + + def test_floor_gates_on_distinct_pages_not_files(self): + root = "/repo/content" + one_page = [f"/repo/content/{lc}/docs/x.md" + for lc in ("de", "ru", "hi", "zh-cn", "es", "pt-br")] + self.assertLessEqual( + len(translate._distinct_orphan_pages(one_page, root)), + translate.ORPHAN_PAGE_FLOOR) + six_pages = [f"/repo/content/de/docs/p{i}.md" for i in range(6)] + self.assertGreater( + len(translate._distinct_orphan_pages(six_pages, root)), + translate.ORPHAN_PAGE_FLOOR) + + +class TestStaleReportCleanup(unittest.TestCase): + def test_empty_worklist_clears_a_prior_run_report(self): + # Once the backlog drains, build_worklist() is empty and main() returns + # early. A report left by an earlier run must still be cleared, or + # run-daily.sh reposts stale findings stamped with today's date. + orig_wl, orig_orph, orig_argv = ( + lib.build_worklist, lib.find_orphan_translations, sys.argv) + existed = os.path.exists(translate.REPORT_PATH) + backup = open(translate.REPORT_PATH).read() if existed else None + try: + lib.build_worklist = lambda *a, **k: [] + lib.find_orphan_translations = lambda *a, **k: [] + sys.argv = ["translate.py"] + with open(translate.REPORT_PATH, "w", encoding="utf-8") as fh: + fh.write("### stale findings from a previous run\n- something\n") + rc = translate.main() + self.assertEqual(rc, 0) + self.assertFalse(os.path.exists(translate.REPORT_PATH), + "stale report was not cleared on an empty worklist") + finally: + lib.build_worklist, lib.find_orphan_translations, sys.argv = ( + orig_wl, orig_orph, orig_argv) + if backup is not None: + with open(translate.REPORT_PATH, "w", encoding="utf-8") as fh: + fh.write(backup) + elif os.path.exists(translate.REPORT_PATH): + os.unlink(translate.REPORT_PATH) + + def test_empty_worklist_still_reports_a_transcreated_page(self): + # The mirror of the above: an empty worklist must NOT suppress the + # transcreated-drift report. A hand-authored page whose English source + # drifted still needs posting, or the report never fires on the steady-state + # empty-worklist path. + orig_wl, orig_orph, orig_tc, orig_argv = ( + lib.build_worklist, lib.find_orphan_translations, lib.find_transcreated, sys.argv) + existed = os.path.exists(translate.REPORT_PATH) + backup = open(translate.REPORT_PATH).read() if existed else None + try: + lib.build_worklist = lambda *a, **k: [] + lib.find_orphan_translations = lambda *a, **k: [] + lib.find_transcreated = lambda *a, **k: [("ru", "docs/x.md")] + sys.argv = ["translate.py"] + rc = translate.main() + self.assertEqual(rc, 0) + self.assertTrue(os.path.exists(translate.REPORT_PATH), + "transcreated report did not fire on the empty-worklist path") + self.assertIn("docs/x.md", open(translate.REPORT_PATH).read()) + finally: + (lib.build_worklist, lib.find_orphan_translations, + lib.find_transcreated, sys.argv) = (orig_wl, orig_orph, orig_tc, orig_argv) + if backup is not None: + with open(translate.REPORT_PATH, "w", encoding="utf-8") as fh: + fh.write(backup) + elif os.path.exists(translate.REPORT_PATH): + os.unlink(translate.REPORT_PATH) + + +class TestLinkDestinationMasking(unittest.TestCase): + """URLs used to be defended by a prompt rule only. They are masked now, so a + model that "fixes" or localizes a link cannot silently ship a broken one.""" + + def test_inline_link_destination_is_masked_but_text_is_not(self): + masked, store = lib.protect("See the [install guide](/docs/v1.5/install) for details.") + self.assertNotIn("/docs/v1.5/install", masked) + self.assertIn("install guide", masked) # link text stays translatable + self.assertEqual(lib.restore(masked, store), + "See the [install guide](/docs/v1.5/install) for details.") + + def test_autolink_and_refdef_are_masked(self): + text = "Visit <https://cozystack.io> now.\n\n[ref]: https://example.com/a?b=c\n" + masked, store = lib.protect(text) + self.assertNotIn("cozystack.io", masked) + self.assertNotIn("example.com", masked) + self.assertEqual(lib.restore(masked, store), text) + + def test_footnote_definition_prose_is_not_masked(self): + # `[^1]: text` is a footnote, not a reference definition: its first word + # is prose and must stay translatable. + masked, store = lib.protect("Body.[^1]\n\n[^1]: Footnote prose here.\n") + self.assertIn("Footnote prose here.", masked) + self.assertEqual(lib.restore(masked, store), "Body.[^1]\n\n[^1]: Footnote prose here.\n") + + def test_link_title_is_left_translatable(self): + masked, store = lib.protect('A [link](/a/b "Read this") here.') + self.assertIn('"Read this"', masked) + self.assertNotIn("/a/b", masked) + self.assertEqual(lib.restore(masked, store), 'A [link](/a/b "Read this") here.') + + +class TestTypographyMarkup(unittest.TestCase): + def test_html_entity_semicolon_is_not_prose_punctuation(self): + self.assertEqual(lib.check_typography("了解更多 →\n\n## 社区", "zh-cn"), []) + self.assertEqual(lib.check_typography("了解更多 → 社区", "zh-cn"), []) + + def test_real_half_width_semicolon_is_still_caught(self): + self.assertTrue(lib.check_typography("了解更多; 社区", "zh-cn")) + + +class TestIntegrityFindings(unittest.TestCase): + def test_clean_translation_has_no_findings(self): + self.assertEqual( + lib.integrity_findings("Cozystack v1.5 uses --dry-run.", + "Cozystack v1.5 verwendet --dry-run.", ["Cozystack"]), []) + + def test_dropped_version_is_major(self): + f = lib.integrity_findings("Upgrade to v1.5 now.", "Jetzt aktualisieren.") + self.assertTrue(any(x["severity"] == "major" and "v1.5" in x["issue"] for x in f)) + + def test_localized_decimal_in_a_version_is_caught(self): + # "v1.5" rewritten as "v1,5" — the token no longer matches the source. + f = lib.integrity_findings("Use v1.5.", "Nutze v1,5.") + self.assertTrue(any("v1.5" in x["issue"] for x in f)) + + def test_dropped_flag_is_major(self): + f = lib.integrity_findings("Pass --dry-run to preview.", "Zum Testen übergeben.") + self.assertTrue(any(x["severity"] == "major" and "--dry-run" in x["issue"] for x in f)) + + def test_translated_brand_is_caught(self): + f = lib.integrity_findings("Cozystack is a platform.", "Козистек — это платформа.", + ["Cozystack"]) + self.assertTrue(any("Cozystack" in x["issue"] for x in f)) + + def test_code_spans_are_exempt(self): + # A version that only lives inside code is already guaranteed by masking; + # it must not be double-reported here. + self.assertEqual(lib.integrity_findings("Run `helm install v1.5`.", "Führen Sie aus."), []) + + def test_prose_decimal_reformat_is_allowed(self): + # The style guides MANDATE the decimal comma in prose; a bare two-part + # decimal is a quantity, not a version, and must not be enforced + # byte-for-byte (the finding would refire every revise round and block + # the gate forever). + self.assertEqual(lib.integrity_findings("It is 3.14 wide.", "Es ist 3,14 breit."), []) + + def test_thousands_separator_reformat_is_allowed(self): + self.assertEqual( + lib.integrity_findings("It runs 10,000 pods.", "Es betreibt 10.000 Pods."), []) + + def test_three_component_bare_version_is_still_caught(self): + f = lib.integrity_findings("Upgrade to 1.2.3 now.", "Jetzt aktualisieren.") + self.assertTrue(any(x["severity"] == "major" and "1.2.3" in x["issue"] for x in f)) + + def test_dnt_term_is_not_counted_inside_larger_words(self): + # Substring counting would see 'Go' inside 'Google' and demand a third + # occurrence the translation never had. + self.assertEqual( + lib.integrity_findings("The Go toolchain is part of Google.", + "Der Go-Toolchain gehört zu Alphabet.", ["Go"]), []) + + def test_localized_date_is_not_an_invented_version(self): + # The de guide mandates 24.07.2026 for numeric dates in prose; the bare + # three-component branch must not read it as an invented version. + self.assertEqual( + lib.integrity_findings("Released on 07/24/2026.", + "Veröffentlicht am 24.07.2026."), []) + + def test_localized_thousands_grouping_is_not_an_invented_version(self): + self.assertEqual( + lib.integrity_findings("It holds 10,000,000 records.", + "Es hält 10.000.000 Einträge."), []) + + +class TestTypographyChecks(unittest.TestCase): + def test_russian_ascii_quotes_flagged(self): + f = lib.check_typography('Это "кластер" здесь.', "ru") + self.assertTrue(any("ёлочки" in x["issue"] for x in f)) + + def test_russian_guillemets_pass(self): + self.assertEqual(lib.check_typography("Это «кластер» здесь.", "ru"), []) + + def test_chinese_halfwidth_punctuation_flagged(self): + self.assertTrue(lib.check_typography("这是集群, 然后部署", "zh-cn")) + + def test_chinese_fullwidth_punctuation_passes(self): + self.assertEqual(lib.check_typography("这是集群，然后部署。", "zh-cn"), []) + + def test_pt_pt_vocabulary_leak_flagged(self): + f = lib.check_typography("Abra o ficheiro de configuração.", "pt-br") + self.assertTrue(any("European Portuguese" in x["issue"] for x in f)) + + def test_spanish_opened_question_with_brand_passes(self): + # The rule must anchor at sentence start; matching at any capitalized + # word would re-match from the brand inside a correctly opened «¿…?». + self.assertEqual(lib.check_typography("¿Qué es Cozystack?", "es"), []) + + def test_spanish_mid_sentence_marks_pass(self): + # The es guide's own ✓ forms: the mark goes where the question or + # exclamation actually starts, which may be mid-sentence. + self.assertEqual( + lib.check_typography("Si el nodo falla, ¿qué pasa con los datos?", "es"), []) + self.assertEqual( + lib.check_typography("Listo, ¡ya tienes un clúster!", "es"), []) + + def test_russian_nested_quotes_pass(self): + # „лапки“ close with U+201C; two nested pairs on one line must not read + # as an English “…” pair. + self.assertEqual( + lib.check_typography("Задайте параметры „replicas“ и „selector“ в манифесте.", "ru"), + []) + + def test_russian_english_curly_pair_still_flagged(self): + f = lib.check_typography("Это “cluster” здесь.", "ru") + self.assertTrue(any("ёлочки" in x["issue"] for x in f)) + + def test_spanish_unopened_question_flagged(self): + f = lib.check_typography("Cómo funciona esto exactamente?", "es") + self.assertTrue(any("¿" in x["issue"] for x in f)) + + def test_spanish_unopened_question_mid_paragraph_flagged(self): + f = lib.check_typography("Listo. Cómo funciona esto exactamente?", "es") + self.assertTrue(any("¿" in x["issue"] for x in f)) + + def test_devanagari_digits_flagged(self): + self.assertTrue(lib.check_typography("क्लस्टर में ३ नोड हैं।", "hi")) + + def test_code_spans_are_exempt_from_typography(self): + # Straight quotes inside code are correct and must not be flagged. + self.assertEqual(lib.check_typography('Запустите `echo "hi"` сейчас.', "ru"), []) + + def test_link_title_quotes_are_not_prose(self): + # A link title's ASCII quotes are CommonMark delimiters. Flagging them + # would have the revise loop «localize» them into guillemets, breaking + # the link — the exact integrity this pipeline exists to guarantee. + self.assertEqual( + lib.check_typography( + 'Смотрите [руководство](/docs/install "Руководство по установке") здесь.', "ru"), + []) + + def test_image_marker_is_not_prose(self): + # The `!` of `![alt](src)` is syntax. Without stripping it, a heading + # ending in a CJK ideograph followed by an image reads as "half-width + # punctuation after a Chinese character", and an inline image in + # Spanish prose reads as an exclamation without ¡. + self.assertEqual(lib.check_typography("## 架构图\n\n![架构](/img/arch.png)", "zh-cn"), []) + self.assertEqual( + lib.check_typography("Vea la imagen ![diagrama](/img/d.png) para más detalles.", "es"), + []) + + def test_unknown_language_is_a_no_op(self): + self.assertEqual(lib.check_typography('Anything "here".', "xx"), []) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/hack/i18n/translate.py b/hack/i18n/translate.py new file mode 100644 index 000000000..a1fd1ad9c --- /dev/null +++ b/hack/i18n/translate.py @@ -0,0 +1,676 @@ +#!/usr/bin/env python3 +"""Translate Cozystack website pages with a full machine review gate. + +Per page, per language, the pipeline runs: + 1. translate — Claude (Opus) with glossary + style guide + 2. back-translate — literal EN round-trip of the translation + + compare — flag meaning drift vs the original English + 3. review (two natives) — technical editor (fluency) + Cozystack maintainer + (technical correctness); both prompted as native + speakers of the target language + 4. revise-if-needed — one revision addressing all findings, then re-check, + bounded by review.max_rounds / back_translation.max_retries + +Nothing is written until the gate finishes, so an interrupted run never leaves a +half-done page. A page that clears the gate is stamped `auto-reviewed`; one that +runs out of revise rounds with findings still open is written but stamped +`auto-reviewed-with-findings`, so the two are always distinguishable. Reviewer +replies that cannot be parsed count as "revise", never as a silent pass. + +Auth: see `auth` in config.yaml — either a Claude subscription via the Claude +Agent SDK (`claude-agent-sdk`; the base `anthropic` SDK cannot use a +subscription), or a metered org API key. + +Daily-until-limit: on a usage-limit error the run stops cleanly (exit 0) and +resumes next day; already-written pages are skipped via source_digest. + +Usage: + translate.py [--lang ru] [--limit N] [--time-budget-min M] [--dry-run] +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import time + +import yaml + +import lib + +PROMPT_DIR = os.path.join(os.path.dirname(__file__), "prompts") +STYLE_DIR = os.path.join(os.path.dirname(__file__), "style-guides") +# Findings still open after the revise loop, for the weekly PR body. Gitignored: +# it is a per-run artifact, not content. +REPORT_PATH = os.path.join(os.path.dirname(__file__), "last-run-findings.md") +# Orphan mass-deletion floor: refuse to auto-delete translations of more than +# this many distinct English pages in one run (a large batch means the English +# tree moved, not that N pages died). Module-level so it is pinned by a test. +ORPHAN_PAGE_FLOOR = 5 + + +def _format_report(report: list[dict]) -> str: + """Render open findings as the markdown that goes in the PR body.""" + out = [f"### Pages with open findings ({len(report)})", "", + "The revise loop ran out of rounds with these findings still raised by at", + "least one reviewer, so the pages are stamped", + "`translation_review: auto-reviewed-with-findings`. They are published anyway", + "(publish-then-ratify) but are the ones worth a human's attention first.", + "", + "Findings come from the model reviewers, so some will be noise — that is", + "expected; they are listed rather than acted on automatically.", ""] + for entry in report: + out.append(f"**`{entry['lang']}: {entry['rel']}`**") + out.append("") + for f in entry["findings"]: + sev, who = f.get("severity", "?"), f.get("from", "?") + out.append(f"- _{sev}_ ({who}): {f.get('issue', f)}") + out.append("") + return "\n".join(out) + + +class RateLimited(Exception): + """Raised when the subscription hits its usage limit — stop the run cleanly.""" + + +def _read(path: str) -> str: + """Read a prompt/style file. A missing prompt is a hard failure — silently + returning "" would send a reviewer out with no instructions, and its prose + reply would then be read as a pass.""" + if not os.path.exists(path): + raise FileNotFoundError(f"required prompt/config file missing: {path}") + return open(path, encoding="utf-8").read() + + +def _read_optional(path: str, default: str = "") -> str: + return open(path, encoding="utf-8").read() if os.path.exists(path) else default + + +def render(template: str, lang_cfg: dict, glossary: dict) -> str: + dnt = ", ".join(glossary.get("do_not_translate", [])) + preferred = "\n".join( + f" - {t} -> {pl[lang_cfg['code']]}" + for t, pl in (glossary.get("preferred") or {}).items() + if lang_cfg["code"] in pl + ) or " (none specified)" + style = _read_optional(os.path.join(STYLE_DIR, f"{lang_cfg['code']}.md"), + f"(use professional, natural {lang_cfg['name']}.)") + return (template + .replace("{{LANGUAGE}}", lang_cfg["name"]) + .replace("{{LANG_CODE}}", lang_cfg["code"]) + .replace("{{DO_NOT_TRANSLATE}}", dnt) + .replace("{{PREFERRED_TERMS}}", preferred) + .replace("{{STYLE_GUIDE}}", style)) + + +# The Agent SDK's rate-limit exception type is version-dependent / undocumented, +# so we match on the message text. Broaden this list if a limit slips through. +# Only markers that mean "the subscription/account is out of budget". Deliberately +# NOT "overloaded" (transient 529) or a bare "429" (can appear in unrelated text): +# treating those as the daily limit would silently end the run with exit 0 and an +# skipped day would look identical to a successful one. +_RATE_LIMIT_MARKERS = ("rate_limit_error", "usage limit", "quota exceeded", + "out of credit", "credit balance") + + +async def _aquery(cfg, system, payload): + """One single-turn, tool-less completion via the Claude Agent SDK (subscription).""" + from claude_agent_sdk import query, ClaudeAgentOptions, AssistantMessage, TextBlock + opts = ClaudeAgentOptions( + model=cfg["model"]["default"], + system_prompt=system, + allowed_tools=[], # pure text: no tools, no filesystem, no MCP, no approval prompts + max_turns=1, + ) + out = [] + async for message in query(prompt=payload, options=opts): + if isinstance(message, AssistantMessage): + for block in message.content: + if isinstance(block, TextBlock): + out.append(block.text) + return "".join(out).strip() + + +def call(cfg, system, payload): + """One Opus call over the Max subscription (Claude Agent SDK / CLAUDE_CODE_OAUTH_TOKEN). + + Raises RateLimited when the subscription's usage limit is hit, so the daily + run stops cleanly and resumes tomorrow. + """ + import asyncio + try: + return asyncio.run(_aquery(cfg, system, payload)) + except RateLimited: + raise + except Exception as exc: + if any(m in str(exc).lower() for m in _RATE_LIMIT_MARKERS): + raise RateLimited(str(exc)) from exc + raise + + +def _parse_verdict(raw: str, who: str) -> dict: + """Pull the JSON verdict out of a reviewer response. + + FAILS CLOSED: anything we cannot parse as a verdict (model refusal, prose, + truncated JSON) counts as "revise", never a silent pass — otherwise a broken + prompt or SDK would wave every page through and the gate would be + unfalsifiable. Tries the non-greedy match first so prose containing braces + before the JSON doesn't swallow the whole response. + """ + for pat in (r"\{.*?\}\s*$", r"\{.*\}"): + m = re.search(pat, raw.strip(), re.DOTALL) + if not m: + continue + try: + v = json.loads(m.group(0)) + except json.JSONDecodeError: + continue + if isinstance(v, dict) and v.get("verdict") in ("pass", "revise"): + v.setdefault("findings", []) + return v + return {"verdict": "revise", "parse_failed": True, + "findings": [{"severity": "major", "from": who, + "issue": f"unparseable reviewer response from {who} " + f"(treated as revise): {raw.strip()[:200]}"}]} + + +class ProtocolError(Exception): + """The model did not answer in the ===FRONTMATTER===/===BODY=== protocol.""" + + +def _split_payload_response(out: str, store: dict, expect: set[str]) -> tuple[dict, str]: + """Parse a translate/revise reply. Raises rather than treating a preamble + ("Here's the translation:") as the page body, and verifies every protected + placeholder survived — a dropped §§FENCE_n§§ would silently delete a code + block from the page. + + The front-matter half is JSON ({path: translated_text}), not `key: value` + lines: values are addressed by path so nested copy (taglines[0], + benefits[1].description) round-trips, and a multi-line description no longer + gets shredded by a line-oriented parser. + """ + if "===BODY===" not in out: + raise ProtocolError(f"no ===BODY=== marker in reply: {out.strip()[:200]}") + head, body = out.split("===BODY===", 1) + head = head.replace("===FRONTMATTER===", "").strip() + if expect: + m = re.search(r"\{.*\}", head, re.DOTALL) + if not m: + raise ProtocolError(f"front-matter half is not JSON: {head[:200]}") + try: + tr_fm = json.loads(m.group(0)) + except json.JSONDecodeError as exc: + raise ProtocolError(f"front-matter JSON did not parse: {exc}") from exc + if not isinstance(tr_fm, dict): + raise ProtocolError("front-matter JSON is not an object") + # Missing keys mean the hero/cards would silently stay English on a page + # whose body IS translated — a half-translated page is worse than a retry. + missing = expect - set(tr_fm) + if missing: + raise ProtocolError(f"front-matter keys not returned: {sorted(missing)}") + tr_fm = {k: v for k, v in tr_fm.items() if k in expect and isinstance(v, str)} + else: + tr_fm = {} + body = body.strip() + # A token the model was actually SENT must survive exactly once; a token it was + # never sent must not appear at all. A token is sent iff it is TOP-LEVEL — not + # contained in another entry's stored value. A nested token (e.g. an SC shortcode + # stashed inside an INLINECODE span — `` `{{< version-pin "x" >}}` ``) never + # appears in the masked text the model sees and is unwound transitively by + # restore()'s fixed-point loop, so it must be 0× here: if it shows up the model + # injected it, and reverse restore would expand it into duplicated content the + # residual check can no longer see. + top_level = {tok for tok in store + if not any(tok in val for other, val in store.items() if other != tok)} + bad = {tok: body.count(tok) for tok in store + if body.count(tok) != (1 if tok in top_level else 0)} + if bad: + tok, n = next(iter(bad.items())) + expected = 1 if tok in top_level else 0 + what = "injected" if expected == 0 else "lost" if n == 0 else "duplicated" + raise ProtocolError(f"{len(bad)} protected placeholder(s) {what} by the model " + f"(e.g. {tok} appears {n}×, expected {expected}) — refusing to write a " + f"page with dropped, duplicated or injected code") + return tr_fm, lib.restore(body, store) + + +def translate_page(cfg, glossary, lang_cfg, rel) -> tuple[str, bool, list[dict]]: + """Run the full gate for one page. + + Returns (file_text, gate_passed, findings). gate_passed is False when the + revise loop ran out of rounds with findings still open — the page is still + written (publish-then-review), but stamped honestly so it is distinguishable + from a page that actually cleared the gate. + + `findings` are the ones still open on the final round. They are returned, not + swallowed: a page stamped `-with-findings` with no record of WHAT was found + tells a maintainer only that something is wrong, which is not reviewable. + """ + src = lib.source_path(cfg, rel) + text = open(src, encoding="utf-8").read() + fm, body, _ = lib.split_frontmatter(text) + if fm is None: + # Unparseable / absent YAML front matter: writing the page would silently + # drop slug/date/aliases/images. Refuse instead. + raise ProtocolError(f"{rel}: could not parse YAML front matter in the English source") + masked_body, store = lib.protect(body) + + # Every user-visible string in the front matter, at any depth, addressed by + # path. The landing page renders its hero and cards from taglines[]/ + # benefits[]/features[]; a top-level-only walk would leave them English. + fm_values = lib.extract_translatable(fm) + expect = set(fm_values) + + _FM_PROTOCOL = ( + "Translate the FRONTMATTER values and the BODY below.\n" + "FRONTMATTER is a JSON object mapping an opaque path to the English text.\n" + "Return a JSON object with the SAME keys and translated values — do not add,\n" + "drop, or rename keys, and do not interpret the keys themselves.\n" + "Return EXACTLY:\n===FRONTMATTER===\n<json object>\n===BODY===\n<translated body>" + ) + + # --- Stage 1: translate --- + sys_translate = render(_read(os.path.join(PROMPT_DIR, "translate.md")), lang_cfg, glossary) + payload = ( + _FM_PROTOCOL + + "\n\n===FRONTMATTER===\n" + json.dumps(fm_values, ensure_ascii=False, indent=2) + + "\n===BODY===\n" + masked_body + ) + tr_fm, tr_body = _split_payload_response(call(cfg, sys_translate, payload), store, expect) + + # --- Stages 2-4: gate with revise loop --- + max_rounds = max(cfg["review"]["max_rounds"], cfg["back_translation"]["max_retries"]) + gate_passed = False + findings: list[dict] = [] + for round_no in range(max_rounds + 1): + findings = [] + + # 2. back-translation round-trip + if cfg["back_translation"]["enabled"]: + masked_tr, tr_store = lib.protect(tr_body) + back_en = call(cfg, + render(_read(os.path.join(PROMPT_DIR, "back-translate.md")), lang_cfg, glossary), + masked_tr) + cmp = _parse_verdict(call(cfg, + render(_read(os.path.join(PROMPT_DIR, "back-translate-compare.md")), lang_cfg, glossary), + f"ORIGINAL:\n{body}\n\nBACK-TRANSLATION:\n{lib.restore(back_en, tr_store)}"), + "back-translation") + if cmp["verdict"] == "revise": + # A "revise" with an empty findings list must still block the gate: + # deriving pass/fail from len(findings) would let an explicit + # "redo this" verdict through as auto-reviewed. + findings += ([{**f, "from": "back-translation"} for f in cmp["findings"]] + or [{"severity": "major", "from": "back-translation", + "issue": "revise verdict with no findings listed"}]) + + # 2b. deterministic checks — cheap, objective, and not subject to the + # reviewers' mood. Masking already guarantees code/shortcodes/URLs; these + # cover what legitimately lives in prose (versions, bare flags, brands) + # and the per-language typography the style guides mandate but nothing + # previously verified. They feed the same revise loop as the reviewers. + findings += lib.integrity_findings(body, tr_body, glossary.get("do_not_translate")) + findings += lib.check_typography(tr_body, lang_cfg["code"]) + + # 3. two native reviewers + for reviewer in cfg["review"]["reviewers"]: + sys_r = render(_read(os.path.join(PROMPT_DIR, os.path.basename(reviewer["prompt"]))), + lang_cfg, glossary) + verdict = _parse_verdict(call(cfg, sys_r, + f"ENGLISH SOURCE:\n{body}\n\n{lang_cfg['name'].upper()} TRANSLATION:\n{tr_body}"), + reviewer["id"]) + if verdict["verdict"] == "revise": + findings += ([{**f, "from": reviewer["id"]} for f in verdict["findings"]] + or [{"severity": "major", "from": reviewer["id"], + "issue": "revise verdict with no findings listed"}]) + + if not findings: + gate_passed = True + break + + # Out of rounds: publish what the reviewers last SAW, stamped + # `-with-findings`. Revising here instead would ship text that nothing + # re-checked, and the findings posted to the weekly PR would describe a + # version of the page that no longer exists — the stamp and the report + # must always refer to the text actually written. + if round_no == max_rounds: + break + + # 4. revise addressing all findings, then re-check + masked_body2, store = lib.protect(tr_body) + sys_rev = render(_read(os.path.join(PROMPT_DIR, "revise.md")), lang_cfg, glossary) + rev_payload = ( + _FM_PROTOCOL + + "\n\nENGLISH SOURCE:\n" + body + + "\n\nCURRENT TRANSLATION:\n===FRONTMATTER===\n" + + json.dumps(tr_fm, ensure_ascii=False, indent=2) + + "\n===BODY===\n" + masked_body2 + + "\n\nFINDINGS:\n" + json.dumps(findings, ensure_ascii=False, indent=2) + ) + tr_fm, tr_body = _split_payload_response(call(cfg, sys_rev, rev_payload), store, expect) + + # --- assemble --- + # Apply by path onto a copy of the English front matter: structural keys + # (slug, date, weight, icon, aliases, images) are carried over untouched. + out_fm = lib.apply_translations(fm, tr_fm) + # Then re-attach keys a human added to the localized page that do not exist + # upstream (a locale-specific `seo:` block, `l10n:` notes). Deriving the page + # purely from the English front matter would delete native reviewers' work + # every time the source changed. + dest = lib.target_path(cfg, lang_cfg["code"], rel) + if os.path.exists(dest): + existing_fm, _, _ = lib.split_frontmatter(open(dest, encoding="utf-8").read()) + out_fm = lib.merge_target_only_keys(out_fm, existing_fm) + out_fm["source_digest"] = f"sha256:{lib.sha256_file(src)}" + # `l10n` is the site's existing convention for HOW a page was localized + # (mt | transcreate). Whatever the page was before, this pipeline just + # machine-translated it, so say so — the disclaimer banner and any future + # native-review triage read this. Defense in depth: build_worklist already + # skips `transcreate` pages, but if translate_page is ever reached on one, + # preserve the hand-authored marker rather than silently downgrading it to + # `mt` and erasing the "a human wrote this" signal. + out_fm["l10n"] = "transcreate" if (os.path.exists(dest) and lib.l10n_mode(dest) == "transcreate") else "mt" + # Stamp honestly: a page that ran out of revise rounds with findings still + # open is NOT the same as one that cleared the gate. Neither is "ratified" — + # only a human sets that, and only that value drops the disclaimer banner. + out_fm[cfg["review_status_field"]] = ( + cfg["review_status_value"] if gate_passed else f"{cfg['review_status_value']}-with-findings") + fm_yaml = yaml.safe_dump(out_fm, allow_unicode=True, sort_keys=False, + default_flow_style=False, width=10 ** 9).strip() + # Rewrite ref/relref cross-links to plain links: a link to a sibling page not + # yet translated into this language would otherwise hard-fail the Hugo build + # (REF_NOT_FOUND). See lib.deref_shortcodes. + tr_body = lib.deref_shortcodes(tr_body, rel) + # Fail closed: if any ref/relref survived (an unsupported shortcode shape), + # refuse to write the page rather than ship one that can break the build. + if lib.has_ref_shortcode(tr_body): + raise ProtocolError( + f"{rel}: a ref/relref shortcode survived deref (unsupported shape) — " + f"refusing to write a page that could break the Hugo build") + return f"---\n{fm_yaml}\n---\n{tr_body}\n", gate_passed, findings + + +def _format_run_status(stopped_early: bool, rate_limit_reason: str, + done: int, skipped: list[dict], attempts: int, + cause: str = "the subscription usage limit") -> str: + """Render an early-stop / skipped-page summary for the weekly PR comment. + + Open findings already have a durable home (the PR comment via last-run-findings.md), + but a run that stopped on the usage limit or skipped a page after repeated + protocol errors did not — it was only ever visible in the run's stdout/stderr, + which a cron/launchd job discards unless the operator captured it. Surfacing it + here gives a maintainer the same durable, actionable record in-band.""" + if not stopped_early and not skipped: + return "" + out = ["### Run status", ""] + if stopped_early: + reason = f" ({rate_limit_reason})" if rate_limit_reason else "" + out.append(f"- Stopped early on {cause} after {done} " + f"page(s){reason}. Remaining pages stay in the worklist and resume " + f"next run.") + if skipped: + out.append(f"- {len(skipped)} page(s) skipped after {attempts} protocol " + f"attempts (left in the worklist, retried next run):") + for s in skipped: + out.append(f" - `{s['lang']}: {s['rel']}` — {s.get('error', '?')}") + out.append("") + return "\n".join(out) + + +def _format_transcreated(pairs: list[tuple[str, str]]) -> str: + """Render the hand-written pages the run deliberately left alone. + + These are the only pages that fall out of scope while their English source + keeps moving, so without a line in the weekly PR they drift silently and + nobody learns the translation needs reworking.""" + if not pairs: + return "" + out = [f"### Hand-written pages skipped ({len(pairs)})", "", + "Their English source changed, but they carry `l10n: transcreate` and were", + "written by a person. The pipeline does not touch them — rework them by hand", + "if the change matters, then refresh `source_digest`.", ""] + for lang, rel in pairs: + out.append(f"- `{lang}: {rel}`") + out.append("") + return "\n".join(out) + + +def _distinct_orphan_pages(orphans: list[str], content_root: str) -> set[str]: + """Collapse orphan translation FILES to distinct English PAGE paths. + + One deleted English page fans out to one orphan per language, so the + mass-deletion floor must count pages, not files (`content/<lang>/<page>` → + `<page>`). Pulled out of main() so the gating comparison is unit-testable.""" + return {os.path.relpath(p, content_root).split(os.sep, 1)[1] for p in orphans} + + +def _budget_exhausted(started: float, budget_min: float | None, now: float) -> bool: + """True once a run has spent its wall-clock budget. + + A scheduled CI job is killed at its timeout and the runner is thrown away, so + pages translated before the kill are lost and paid for again the next day. + Checking the budget before each page lets the run stop early enough for + run-daily.sh to commit and push what it has. A page already in flight is + allowed to finish, so the budget must leave room for one slow page.""" + return budget_min is not None and budget_min > 0 and now - started >= budget_min * 60 + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--lang") + ap.add_argument("--limit", type=int) + ap.add_argument("--path", help="translate only this exact source rel " + "(e.g. docs/v1.5/getting-started/install-kubernetes.md)") + ap.add_argument("--time-budget-min", type=float, + default=float(os.environ["I18N_TIME_BUDGET_MIN"]) + if os.environ.get("I18N_TIME_BUDGET_MIN") else None, + help="stop taking new pages after this many minutes " + "(default: $I18N_TIME_BUDGET_MIN, unset = no budget)") + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args() + + cfg = lib.load_config() + glossary = lib.load_glossary() + lang_by_code = {l["code"]: l for l in cfg["languages"]} + if args.lang and args.lang not in lang_by_code: + print(f"::error::unknown --lang '{args.lang}'; configured: " + f"{', '.join(lang_by_code)}", file=sys.stderr) + return 1 + # The English tree is the reference for BOTH the worklist and orphan + # detection. If it is missing (broken checkout, sparse clone, renamed dir), + # os.walk would silently yield nothing: the worklist would look "done" and + # every stamped translation would look orphaned. Refuse instead. + en_root = os.path.join(lib.REPO_ROOT, cfg["content_dir"], cfg["source_lang"]) + if not os.path.isdir(en_root) or not os.listdir(en_root): + print(f"::error::English content root missing or empty: {en_root} — " + f"refusing to run against a broken checkout", file=sys.stderr) + return 1 + + # Orphans first: a deleted English page must take its translations with it, + # or they outlive the source forever (the worklist only iterates English + # sources). Only `source_digest`-stamped (pipeline-managed) files qualify. + orphans = [] if args.path else lib.find_orphan_translations(cfg, only_lang=args.lang) + content_root = os.path.join(lib.REPO_ROOT, cfg["content_dir"]) + # Two orphan classes with different risk profiles, and the floor must apply to + # only one of them: + # - deleted-source: the English page is gone. A LARGE batch here means the + # English tree moved out from under us (restructure, bad checkout), and + # deleting on that signal would commit a massacre — this is what the floor + # guards. It counts distinct English PAGES, not files (one page fans out to + # one orphan per language). + # - superseded-version: a stamped translation of a non-latest docs version. + # A version bump legitimately orphans a whole retired version at once, so + # this class must NOT count toward the floor — otherwise one release freezes + # ALL orphan cleanup, including genuine source deletions, until someone + # deletes the files by hand. + def _is_deleted_source(path): + rel = os.path.relpath(path, content_root).split(os.sep, 1)[1] + return not os.path.exists(lib.source_path(cfg, rel)) + deleted_source = [p for p in orphans if _is_deleted_source(p)] + superseded = [p for p in orphans if not _is_deleted_source(p)] + ds_pages = _distinct_orphan_pages(deleted_source, content_root) + if len(ds_pages) > ORPHAN_PAGE_FLOOR: + print(f"::warning::{len(deleted_source)} orphaned translations of {len(ds_pages)} " + f"English pages with a deleted source (> {ORPHAN_PAGE_FLOOR} pages) — refusing " + f"to mass-delete; the English tree may have moved. If this is a deliberate " + f"restructure, remove them manually; first few: " + + ", ".join(os.path.relpath(p, lib.REPO_ROOT) for p in deleted_source[:3]), + file=sys.stderr) + deleted_source = [] + # Superseded-version orphans are always safe to remove (a retired docs version). + orphans = deleted_source + superseded + for path in orphans: + rel = os.path.relpath(path, content_root).split(os.sep, 1)[1] + reason = ("English source deleted" + if not os.path.exists(lib.source_path(cfg, rel)) + else "superseded docs version") + if args.dry_run: + print(f" [orphan ] would remove {os.path.relpath(path, lib.REPO_ROOT)} ({reason})") + else: + os.unlink(path) + print(f" ✗ removed orphan {os.path.relpath(path, lib.REPO_ROOT)} ({reason})") + + items = lib.build_worklist(cfg, only_lang=args.lang) + if args.path: + items = [it for it in items if it.rel == args.path] + if not items: + print(f"::error::--path '{args.path}' matched nothing in the worklist " + f"(already translated, out of scope, or misspelled)", file=sys.stderr) + return 1 + if args.limit: + items = items[: args.limit] + + print(f"{len(items)} page(s) to process{' (dry run)' if args.dry_run else ''}") + if args.dry_run: + for it in items: + print(f" [{it.reason:7}] {it.lang}: {it.rel}") + return 0 + if not items: + # An empty worklist is the steady state once the backlog drains, but + # hand-authored `transcreate` pages whose English source has drifted still + # need reporting — build that report here too, otherwise it never fires on + # the empty-worklist path and run-daily.sh has nothing to post. Only fall + # back to clearing a prior run's report when there is genuinely nothing to + # say (else stale findings get reposted with today's date). + transcreated_md = _format_transcreated( + [] if args.path else lib.find_transcreated(cfg, only_lang=args.lang)) + if transcreated_md: + with open(REPORT_PATH, "w", encoding="utf-8") as fh: + fh.write(transcreated_md) + print(f"\nrun report written to {REPORT_PATH}") + elif os.path.exists(REPORT_PATH): + os.unlink(REPORT_PATH) + return 0 + + try: + import claude_agent_sdk # noqa: F401 + except ImportError: + print("::error::claude-agent-sdk not installed (pip install claude-agent-sdk)", file=sys.stderr) + return 1 + # Auth mode is a config decision, not a code one (see config.yaml `auth`). + # I18N_AUTH overrides it for CI, which runs on an org api-key while the tracked + # default stays oauth-subscription for local bootstrap; run-daily.sh reads the + # same var, so the two never disagree. + auth_mode = os.environ.get("I18N_AUTH") or cfg.get("auth", "oauth-subscription") + has_key = bool(os.environ.get("ANTHROPIC_API_KEY")) + if auth_mode == "oauth-subscription" and has_key: + print("::error::auth=oauth-subscription but ANTHROPIC_API_KEY is set — it shadows the " + "subscription and would bill metered API. `unset ANTHROPIC_API_KEY`, or set " + "auth: api-key in config.yaml if metered billing is intended.", file=sys.stderr) + return 1 + if auth_mode == "api-key" and not has_key: + print("::error::auth=api-key but ANTHROPIC_API_KEY is not set.", file=sys.stderr) + return 1 + # Under oauth-subscription, a missing CLAUDE_CODE_OAUTH_TOKEN is fine when the + # `claude` CLI is logged in on this machine — the Agent SDK picks that up. The + # token is only needed headless/CI; if neither exists the SDK raises. + + # A protocol error is usually the model non-deterministically dropping a + # placeholder on a placeholder-dense page (a long docs page can have 60+ + # opaque §§ tokens). It tends to succeed on a fresh attempt, so retry within + # the run rather than only next day — otherwise such a page can fail every + # daily run forever, never publishing while re-spending quota each time. + PROTOCOL_ATTEMPTS = 3 + + clean = with_findings = failed = 0 + stopped_early = False + rate_limit_reason = "" + stop_cause = "the subscription usage limit" + started = time.monotonic() + report: list[dict] = [] + skipped: list[dict] = [] + for it in items: + if _budget_exhausted(started, args.time_budget_min, time.monotonic()): + stopped_early = True + stop_cause = f"the run's {args.time_budget_min:g}-minute time budget" + print(f"\ntime budget of {args.time_budget_min:g} min reached after " + f"{clean + with_findings} page(s) — stopping so the run can commit.") + break + try: + for attempt in range(1, PROTOCOL_ATTEMPTS + 1): + try: + result, gate_passed, findings = translate_page( + cfg, glossary, lang_by_code[it.lang], it.rel) + break + except ProtocolError as exc: + if attempt == PROTOCOL_ATTEMPTS: + raise + print(f" … retry {attempt}/{PROTOCOL_ATTEMPTS - 1} " + f"{it.lang}: {it.rel} — {exc}", file=sys.stderr) + except RateLimited as exc: + stopped_early = True + rate_limit_reason = str(exc) + print(f"\nsubscription usage limit reached after " + f"{clean + with_findings} page(s) — stopping cleanly, resume tomorrow. ({exc})") + break + except ProtocolError as exc: + # Still malformed after every attempt: skip it. It stays in the + # worklist (nothing written, no digest stamped) and is retried next + # run — but the daily run is not stalled on it. + failed += 1 + skipped.append({"lang": it.lang, "rel": it.rel, "error": str(exc)}) + print(f"::warning::skipped {it.lang}: {it.rel} after {PROTOCOL_ATTEMPTS} " + f"attempts — {exc}", file=sys.stderr) + continue + dest = lib.target_path(cfg, it.lang, it.rel) + os.makedirs(os.path.dirname(dest), exist_ok=True) + with open(dest, "w", encoding="utf-8") as fh: + fh.write(result) + if gate_passed: + clean += 1 + print(f" ✓ gate passed {it.lang}: {it.rel}") + else: + with_findings += 1 + print(f" ! findings open {it.lang}: {it.rel} (stamped -with-findings)") + for f in findings: + print(f" [{f.get('severity', '?')}/{f.get('from', '?')}] " + f"{f.get('issue', f)}") + report.append({"lang": it.lang, "rel": it.rel, "findings": findings}) + + # The report is the pipeline's only durable, maintainer-facing channel + # (run-daily.sh posts it as a PR comment). It carries open findings AND an + # early-stop / skipped-page summary — dropping either on the floor leaves a + # `-with-findings` stamp or a stalled page unactionable. + status_md = _format_run_status(stopped_early, rate_limit_reason, + clean + with_findings, skipped, PROTOCOL_ATTEMPTS, + cause=stop_cause) + transcreated_md = _format_transcreated( + [] if args.path else lib.find_transcreated(cfg, only_lang=args.lang)) + sections = [s for s in (status_md, transcreated_md, + _format_report(report) if report else "") if s] + if sections: + with open(REPORT_PATH, "w", encoding="utf-8") as fh: + fh.write("\n".join(sections)) + print(f"\nrun report written to {REPORT_PATH}") + elif os.path.exists(REPORT_PATH): + os.unlink(REPORT_PATH) # don't let a previous run's report go stale + + print(f"\nthis run: {clean} clean, {with_findings} written with open findings, " + f"{failed} skipped on error") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/hack/i18n/worklist.py b/hack/i18n/worklist.py new file mode 100644 index 000000000..94501e65c --- /dev/null +++ b/hack/i18n/worklist.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Print the translation worklist: pages that are missing or stale per language. + +Usage: + worklist.py # all languages, human-readable summary + worklist.py --json # machine-readable + worklist.py --lang ru # restrict to one language + worklist.py --limit 3 # only the first N items (mirrors translate.py) + worklist.py --path blog/x.md # restrict to one source page + +"Missing" = no translated file exists. "Stale" = the translated file's +source_digest no longer matches the current sha256 of its English source. +Exits 0 with work to do or none (an empty list means everything is up to date); +exits 1 only on a bad argument. + +The flags mirror translate.py's on purpose: run-daily.sh forwards the same "$@" +to both, so this script previews exactly what that run will translate. +""" + +from __future__ import annotations + +import argparse +import json +import sys + +import lib + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--json", action="store_true", help="emit JSON") + ap.add_argument("--lang", help="restrict to one language code") + ap.add_argument("--limit", type=int, help="only the first N items") + ap.add_argument("--path", help="restrict to this exact source rel " + "(e.g. docs/v1.5/getting-started/install-kubernetes.md)") + ap.add_argument("--dry-run", action="store_true", + help="accepted for parity with translate.py; this script never writes") + args = ap.parse_args() + + cfg = lib.load_config() + codes = [l["code"] for l in cfg["languages"]] + if args.lang and args.lang not in codes: + # Without this, a typo filters everything out and prints "all languages + # up to date" — the same reassuring lie as an empty worklist. + print(f"::error::unknown --lang '{args.lang}'; enabled: {', '.join(codes)}", + file=sys.stderr) + return 1 + items = lib.build_worklist(cfg, only_lang=args.lang) + if args.path: + items = [i for i in items if i.rel == args.path] + if args.limit: + items = items[: args.limit] + + if args.json: + json.dump( + [{"lang": i.lang, "rel": i.rel, "reason": i.reason} for i in items], + sys.stdout, + ) + sys.stdout.write("\n") + return 0 + + if not items: + print("translation worklist: empty — all languages up to date") + return 0 + + by_lang: dict[str, list[lib.WorkItem]] = {} + for i in items: + by_lang.setdefault(i.lang, []).append(i) + total = len(items) + print(f"translation worklist: {total} page(s) need work\n") + for lang in sorted(by_lang): + rows = by_lang[lang] + missing = sum(1 for r in rows if r.reason == "missing") + stale = sum(1 for r in rows if r.reason == "stale") + print(f" {lang}: {len(rows)} ({missing} missing, {stale} stale)") + for r in rows: + print(f" [{r.reason:7}] {r.rel}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/hugo.yaml b/hugo.yaml index 25f8eb684..f773a9049 100644 --- a/hugo.yaml +++ b/hugo.yaml @@ -3,19 +3,19 @@ title: Cozystack enableRobotsTXT: true enableEmoji: true -# Since Hugo v0.161.0 the Node asset tools (PostCSS here) run under Node's -# permission model, with read access limited to the project directory by -# default. That default is not enough for this site: the Docsy theme arrives as -# a Hugo module and its SCSS is served from the module cache outside the project -# tree, so the CSS build fails with "Use --allow-fs-read to manage permissions". -# -# Read access is therefore widened. Write access stays denied (allowWrite is -# left at its empty default), as do addons, child processes and workers beyond -# what Hugo already permits for tailwindcss. -# -# This is deliberately broader than necessary — the minimal set of readable -# paths has not been pinned down, and narrowing it would be a welcome follow-up. security: + # Since Hugo v0.161.0 the Node asset tools (PostCSS here) run under Node's + # permission model, with read access limited to the project directory by + # default. That default is not enough for this site: the Docsy theme arrives as + # a Hugo module and its SCSS is served from the module cache outside the project + # tree, so the CSS build fails with "Use --allow-fs-read to manage permissions". + # + # Read access is therefore widened. Write access stays denied (allowWrite is + # left at its empty default), as do addons, child processes and workers beyond + # what Hugo already permits for tailwindcss. + # + # This is deliberately broader than necessary — the minimal set of readable + # paths has not been pinned down, and narrowing it would be a welcome follow-up. node: permissions: allowRead: @@ -109,6 +109,16 @@ module: lang: hi files: - '! **/_include/*' + - source: content/es + target: content + lang: es + files: + - '! **/_include/*' + - source: content/pt-br + target: content + lang: pt-br + files: + - '! **/_include/*' # Mount npm dependencies for Docsy - source: node_modules/bootstrap target: assets/vendor/bootstrap @@ -180,6 +190,31 @@ languages: params: time_format_default: 2006-01-02 time_format_blog: 2006-01-02 + # es and pt-br are being translated (they are listed in hack/i18n/config.yaml) + # but are NOT declared here yet, because content/es and content/pt-br do not + # exist. Declaring a language with no content does not produce nothing — Hugo + # still emits /es/, /es/tags/, /es/categories/, /es/topics/, + # /es/article_types/ and /es/404.html, all `robots: index, follow` and + # self-canonical. That publishes ~6 empty but indexable pages per language, + # which is the thin content we are otherwise careful to avoid. + # + # Un-comment each block in the same PR that lands that language's content — + # not before. hack/i18n/test_i18n.py enforces this. + # + # es: + # languageName: Español + # languageCode: es + # weight: 6 + # params: + # time_format_default: 2006-01-02 + # time_format_blog: 2006-01-02 + # pt-br: + # languageName: Português (Brasil) + # languageCode: pt-BR + # weight: 7 + # params: + # time_format_default: 2006-01-02 + # time_format_blog: 2006-01-02 permalinks: blog: /:section/:year/:month/:slug/ diff --git a/i18n/de.toml b/i18n/de.toml index 6c590cd72..d7da2e0a9 100644 --- a/i18n/de.toml +++ b/i18n/de.toml @@ -84,3 +84,11 @@ other = "Aktuell (stabil)" [docs_landing_status_previous] other = "Frühere Version" +[translation_machine_notice] +other = "Diese Seite wurde maschinell übersetzt und noch nicht von einem Muttersprachler geprüft." + +[translation_report_error] +other = 'Einen Fehler entdeckt? <a href="{{ .IssueURL }}">Eröffnen Sie ein Issue</a>.' + +[translation_see_original] +other = 'Siehe das <a href="{{ .Link }}">englische Original</a>.' diff --git a/i18n/en.toml b/i18n/en.toml index 9b3b6f7fd..05d368007 100644 --- a/i18n/en.toml +++ b/i18n/en.toml @@ -84,3 +84,11 @@ other = "Latest (stable)" [docs_landing_status_previous] other = "Previous release" +[translation_machine_notice] +other = "This page was machine-translated and has not yet been reviewed by a native speaker." + +[translation_report_error] +other = 'Spotted a mistake? <a href="{{ .IssueURL }}">Open an issue</a>.' + +[translation_see_original] +other = 'See the <a href="{{ .Link }}">English original</a>.' diff --git a/i18n/es.toml b/i18n/es.toml new file mode 100644 index 000000000..7ff68ad2e --- /dev/null +++ b/i18n/es.toml @@ -0,0 +1,95 @@ +[caution] +other = "Precaución:" + +[home_screenshots] +other = "Capturas de pantalla" + +[home_benefits] +other = "Beneficios" + +[home_features] +other = "Funcionalidades" + +[home_cncf_sandbox] +other = "Cozystack es un proyecto Sandbox de la <a href=\"https://cncf.io\">Cloud Native Computing Foundation</a>" + +[join_community] +other = "Únete a la comunidad" + +[docs_create_issue] +other = "Crear incidencia de documentación" + +[docs_create_project_issue] +other = "Crear incidencia del proyecto" + +[docs_edit] +other = "Editar esta página" + +[docs_last_mod] +other = "Última modificación" + +[docs_table_of_contents] +other = "Tabla de contenidos" + +[note] +other = "Nota:" + +[outdated_blog__message] +other = "Este artículo tiene más de un año. Los artículos más antiguos pueden contener contenido desactualizado. Comprueba que la información de la página no se haya vuelto incorrecta desde su publicación." + +[ui_minute_read] +other = "minutos de lectura" + +[ui_pager_next] +other = "Siguiente" + +[ui_pager_prev] +other = "Anterior" + +[ui_read_more] +other = "Leer más" + +[warning] +other = "Advertencia:" + +[translation_machine_notice] +other = "Esta página se tradujo automáticamente y aún no ha sido revisada por un hablante nativo." + +[translation_report_error] +other = '¿Has encontrado un error? <a href="{{ .IssueURL }}">Abre un issue</a>.' + +[translation_see_original] +other = 'Consulta el <a href="{{ .Link }}">original en inglés</a>.' + +[docs_landing_new_users] +other = "<strong>Nuevos usuarios:</strong> empiece con la <a href=\"{{ .url }}\">documentación de {{ .version }}</a>, la versión estable actual." + +[docs_landing_available_versions] +other = "Versiones disponibles" + +[docs_landing_col_version] +other = "Versión" + +[docs_landing_col_status] +other = "Estado" + +[docs_landing_status_latest] +other = "Última (estable)" + +[docs_landing_status_previous] +other = "Versión anterior" + +[home_use_cases] +other = "Dónde encaja Cozystack" + +[home_use_cases_lead] +other = "Seis situaciones para las que se creó el proyecto. En cada una de ellas hay alguien que ya lo ejecuta en producción hoy." + +[home_use_cases_more] +other = "Leer más" + +[home_use_cases_prev] +other = "Caso de uso anterior" + +[home_use_cases_next] +other = "Caso de uso siguiente" diff --git a/i18n/hi.toml b/i18n/hi.toml index 2975342de..865297119 100644 --- a/i18n/hi.toml +++ b/i18n/hi.toml @@ -84,3 +84,11 @@ other = "नवीनतम (स्थिर)" [docs_landing_status_previous] other = "पिछली रिलीज़" +[translation_machine_notice] +other = "यह पृष्ठ मशीन द्वारा अनूदित है और अभी तक किसी मूल वक्ता द्वारा जाँचा नहीं गया है।" + +[translation_report_error] +other = 'कोई त्रुटि मिली? <a href="{{ .IssueURL }}">Issue बनाएँ</a>।' + +[translation_see_original] +other = '<a href="{{ .Link }}">अंग्रेज़ी मूल</a> देखें।' diff --git a/i18n/pt-br.toml b/i18n/pt-br.toml new file mode 100644 index 000000000..5150b8fee --- /dev/null +++ b/i18n/pt-br.toml @@ -0,0 +1,95 @@ +[caution] +other = "Cuidado:" + +[home_screenshots] +other = "Capturas de tela" + +[home_benefits] +other = "Benefícios" + +[home_features] +other = "Recursos" + +[home_cncf_sandbox] +other = "O Cozystack é um projeto Sandbox da <a href=\"https://cncf.io\">Cloud Native Computing Foundation</a>" + +[join_community] +other = "Junte-se à comunidade" + +[docs_create_issue] +other = "Criar issue de documentação" + +[docs_create_project_issue] +other = "Criar issue do projeto" + +[docs_edit] +other = "Editar esta página" + +[docs_last_mod] +other = "Última modificação" + +[docs_table_of_contents] +other = "Sumário" + +[note] +other = "Nota:" + +[outdated_blog__message] +other = "Este artigo tem mais de um ano. Artigos mais antigos podem conter conteúdo desatualizado. Verifique se as informações da página não se tornaram incorretas desde a sua publicação." + +[ui_minute_read] +other = "minutos de leitura" + +[ui_pager_next] +other = "Próximo" + +[ui_pager_prev] +other = "Anterior" + +[ui_read_more] +other = "Leia mais" + +[warning] +other = "Aviso:" + +[translation_machine_notice] +other = "Esta página foi traduzida automaticamente e ainda não foi revisada por um falante nativo." + +[translation_report_error] +other = 'Encontrou um erro? <a href="{{ .IssueURL }}">Abra uma issue</a>.' + +[translation_see_original] +other = 'Consulte o <a href="{{ .Link }}">original em inglês</a>.' + +[docs_landing_new_users] +other = "<strong>Novos usuários:</strong> comece pela <a href=\"{{ .url }}\">documentação da {{ .version }}</a> — a versão estável atual." + +[docs_landing_available_versions] +other = "Versões disponíveis" + +[docs_landing_col_version] +other = "Versão" + +[docs_landing_col_status] +other = "Status" + +[docs_landing_status_latest] +other = "Mais recente (estável)" + +[docs_landing_status_previous] +other = "Versão anterior" + +[home_use_cases] +other = "Onde o Cozystack se encaixa" + +[home_use_cases_lead] +other = "Seis situações para as quais o projeto foi criado. Em cada uma delas há alguém executando-o em produção hoje." + +[home_use_cases_more] +other = "Leia mais" + +[home_use_cases_prev] +other = "Caso de uso anterior" + +[home_use_cases_next] +other = "Próximo caso de uso" diff --git a/i18n/ru.toml b/i18n/ru.toml index d188be97f..0eaa8b6fb 100644 --- a/i18n/ru.toml +++ b/i18n/ru.toml @@ -84,3 +84,11 @@ other = "Актуальная (стабильная)" [docs_landing_status_previous] other = "Предыдущий выпуск" +[translation_machine_notice] +other = "Эта страница переведена машинным способом и ещё не проверена носителем языка." + +[translation_report_error] +other = 'Нашли ошибку? <a href="{{ .IssueURL }}">Создайте issue</a>.' + +[translation_see_original] +other = 'Смотрите <a href="{{ .Link }}">английский оригинал</a>.' diff --git a/i18n/zh-cn.toml b/i18n/zh-cn.toml index 37c9a63ff..4252f0c8c 100644 --- a/i18n/zh-cn.toml +++ b/i18n/zh-cn.toml @@ -84,3 +84,11 @@ other = "最新（稳定版）" [docs_landing_status_previous] other = "旧版本" +[translation_machine_notice] +other = "本页面为机器翻译，尚未经过母语者审校。" + +[translation_report_error] +other = '发现错误？<a href="{{ .IssueURL }}">提交 issue</a>。' + +[translation_see_original] +other = '请参阅<a href="{{ .Link }}">英文原文</a>。' diff --git a/layouts/docs/baseof.html b/layouts/docs/baseof.html index 56975ac7c..0eb50a077 100644 --- a/layouts/docs/baseof.html +++ b/layouts/docs/baseof.html @@ -36,6 +36,7 @@ <main class="col-12 col-md-12 col-xl-10 pl-md-5 " role="main"> {{ end }} {{ partial "version-banner.html" . }} + {{ partial "translation-banner.html" . }} {{ if not .Site.Params.ui.breadcrumb_disable }}{{ partial "breadcrumb.html" . }}{{ end }} {{ block "main" . }}{{ end }} </main> diff --git a/layouts/partials/hooks/head-end.html b/layouts/partials/hooks/head-end.html index 4569963d9..984ae3096 100644 --- a/layouts/partials/hooks/head-end.html +++ b/layouts/partials/hooks/head-end.html @@ -13,10 +13,14 @@ {{/* SEO: canonical + noindex for legacy doc versions. Older non-latest doc versions (v0, v1.0, v1.1, …) duplicate content from the latest version (v1.2). Without dedup signals, search engines split - ranking signals across all five copies. We mark old versions as noindex + ranking signals across every copy. We mark old versions as noindex (still crawlable for users following links) and canonicalize current pages to themselves. */}} -{{- $latestVersion := .Site.Params.latest_version_id | default "v1.2" -}} +{{- $latestVersion := .Site.Params.latest_version_id -}} +{{/* Required: an empty value would make (ne .id $latestVersion) true for EVERY + version, noindex-ing the entire docs tree including the latest. Fail the + build loudly rather than silently de-indexing everything. */}} +{{- if not $latestVersion -}}{{- errorf "head-end.html: Site.Params.latest_version_id is required for docs noindex logic but is unset" -}}{{- end -}} {{- $isOldDocsVersion := false -}} {{- range .Site.Params.versions -}} {{- if and .id (ne .id $latestVersion) (ne .id "next") -}} @@ -37,6 +41,19 @@ <link rel="canonical" href="{{ .Permalink }}" /> {{- end }} +{{/* SEO: hreflang alternates. Emitted only when a page has translations, so + search engines and AI crawlers map the per-language versions to each other. + Language codes come from `languageCode` in hugo.yaml (proper BCP-47: + zh-Hans, pt-BR, …). English doubles as x-default. */}} +{{- if gt (len .AllTranslations) 1 }} +{{- range .AllTranslations }} +<link rel="alternate" hreflang="{{ .Language.LanguageCode }}" href="{{ .Permalink }}" /> +{{- if eq .Language.Lang "en" }} +<link rel="alternate" hreflang="x-default" href="{{ .Permalink }}" /> +{{- end }} +{{- end }} +{{- end }} + {{/* JSON-LD Organization (every page) */}} <script type="application/ld+json"> { diff --git a/layouts/partials/translation-banner.html b/layouts/partials/translation-banner.html new file mode 100644 index 000000000..ab21561ac --- /dev/null +++ b/layouts/partials/translation-banner.html @@ -0,0 +1,60 @@ +{{- /* Machine-translation disclaimer. DOCS PAGES ONLY. + + Wired into docs/baseof.html and nowhere else, by product decision: the notice + belongs where a wrong technical detail is costly (an operator following a + translated command), not on the blog, marketing pages, or the homepage hero. + To change coverage, add/remove the `{{ partial "translation-banner.html" . }}` + call in the relevant layout — the guard below is layout-agnostic. + + Localized pages are published as soon as they clear the automated review gate + and are indexed immediately (publish-then-ratify) — readers get the docs now + rather than after a native review that may take months. The trade is that a + reader can land on a page no human has ratified, so the page must say so + plainly and link to the English source it was derived from. + + The rule is fail-safe: every non-English page carries the disclaimer UNLESS it + is explicitly marked ratified. Keying off the presence of `translation_review` + instead would silently exempt the pages translated before this pipeline existed + (content/de, content/ru, content/hi, content/zh-cn from the i18n PoC) — exactly + the unratified machine output the disclaimer is for. + + `translation_review` is written by hack/i18n/translate.py: + auto-reviewed — cleared the machine gate + auto-reviewed-with-findings — written with findings still open + ratified — a native speaker signed off; no banner + Set it to `ratified` on a page a native speaker has checked (or hand-wrote). +*/ -}} +{{- /* $src is the default content language (en). `site.Sites.Default` is the + documented way to reach the default site from any of them; its + `.Language.Lang` is the code we compare against. + + NOT .Site.DefaultContentLanguage — that field does not exist on + page.Site, and referencing it aborts the whole build with + "can't evaluate field DefaultContentLanguage in type page.Site". + Verified against Hugo 0.160.1 (the pinned version) and 0.163. + + NOT .Site.Home.Lang either — on the Russian site that resolves to the + Russian home page, so `ne .Lang .Site.Home.Lang` is false everywhere and + the banner never renders on the very pages it exists for. */ -}} +{{- $src := site.Sites.Default.Language.Lang -}} +{{- if and (ne .Lang $src) (ne .Params.translation_review "ratified") -}} +{{- $en := "" -}} +{{- range .AllTranslations -}} + {{- if eq .Lang $src }}{{ $en = .Permalink }}{{ end -}} +{{- end -}} +{{- /* One i18n key per SENTENCE, with the link as a template placeholder. + Assembling a sentence from fragments ("See the" + <a>…</a> + ".") + hardcodes English word order and punctuation into the layout: Hindi is + verb-final ("अंग्रेज़ी मूल देखें", not "देखें अंग्रेज़ी मूल"), and a fragment + that continues after "?" cannot know it must start a new sentence. + Sentence-level keys let each translator place the link and punctuation + where the language needs them. These UI strings never pass through the + pipeline's review gate (only content/ does) — they are reviewed by hand. */ -}} +<div class="pageinfo pageinfo-warning"> + <p> + {{ T "translation_machine_notice" }} + {{- with $en }} {{ i18n "translation_see_original" (dict "Link" .) | safeHTML }}{{ end }} + {{ i18n "translation_report_error" (dict "IssueURL" "https://github.com/cozystack/website/issues/new") | safeHTML }} + </p> +</div> +{{- end -}} diff --git a/layouts/partials/version-switcher.html b/layouts/partials/version-switcher.html index 5c8037c51..02e16d8e3 100644 --- a/layouts/partials/version-switcher.html +++ b/layouts/partials/version-switcher.html @@ -1,10 +1,22 @@ +{{- /* Language-aware version switcher. + + params.versions[].url are default-language paths (/docs/vX/…). On a localized page + .Page.RelPermalink carries the language prefix (/ru/docs/vX/…), so the raw prefix + match below never fires there and every switch drops the reader onto the English + page. Strip the language base before matching, then link to a localized page when + one is built and to the English page otherwise. + + $langBase is "" for the default language and "/ru", "/zh-cn", "/es" … for the others, + taken from the language's own home permalink. */ -}} +{{- $langBase := strings.TrimSuffix "/" .Site.Home.RelPermalink -}} {{- if and .Site.Params.versions (gt (len .Site.Params.versions) 1) -}} {{- $currentPath := .Page.RelPermalink -}} +{{- $logicalPath := strings.TrimPrefix $langBase $currentPath -}} {{- $currentVersionID := "" -}} {{- $currentVersionUrl := "" -}} {{- $currentVersionLabel := "" -}} {{- range .Site.Params.versions -}} - {{- if hasPrefix $currentPath .url -}} + {{- if hasPrefix $logicalPath .url -}} {{- $currentVersionID = .id -}} {{- $currentVersionUrl = .url -}} {{- $currentVersionLabel = .version -}} @@ -24,11 +36,23 @@ <ul class="version-switcher__menu dropdown-menu" aria-labelledby="versionSwitcherButton"> {{- range .Site.Params.versions }} {{- if not .hidden }} + {{- /* Each target is the most specific page that is actually built, so no + link 404s. The pipeline translates only the latest version, so most + localized pages and version roots never exist. Staying on the same + topic matters more than staying in the language, so the order is: + English version root (always built), localized version root, the same + page in English, the same page localized. site.GetPage resolves in the + current-language site; hugo.Sites.Default is the English site. */ -}} {{- $targetPath := .url -}} + {{- with site.GetPage (strings.TrimRight "/" .url) -}} + {{- $targetPath = .RelPermalink -}} + {{- end -}} {{- if $currentVersionUrl -}} - {{- $subPath := strings.TrimPrefix $currentVersionUrl $currentPath -}} - {{- $candidatePath := printf "%s%s" .url $subPath -}} - {{- $trimmed := strings.TrimRight "/" $candidatePath -}} + {{- $subPath := strings.TrimPrefix $currentVersionUrl $logicalPath -}} + {{- $trimmed := strings.TrimRight "/" (printf "%s%s" .url $subPath) -}} + {{- with hugo.Sites.Default.GetPage $trimmed -}} + {{- $targetPath = .RelPermalink -}} + {{- end -}} {{- with site.GetPage $trimmed -}} {{- $targetPath = .RelPermalink -}} {{- end -}} @@ -44,8 +68,9 @@ </div> {{- else if .Site.Params.versions -}} {{- $currentPath := .Page.RelPermalink -}} +{{- $logicalPath := strings.TrimPrefix $langBase $currentPath -}} {{- range .Site.Params.versions -}} - {{- if hasPrefix $currentPath .url -}} + {{- if hasPrefix $logicalPath .url -}} <span class="version-switcher__label">{{ .version }}</span> {{- end -}} {{- end -}}