From 7c6c77a89ee01163434a5064a2bbcaee4237ce7a Mon Sep 17 00:00:00 2001 From: Will Eastler <32074613+weastler@users.noreply.github.com> Date: Fri, 6 Nov 2026 10:18:12 -0800 Subject: [PATCH 1/6] feat(infrastructure): add Azure deployment foundations Add optional registry/vault and private-network provisioning modules, native release safety gates, and proxy-backed image builds. Preserve the latest Hub admission and session configuration in deployment guidance. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/hub-infrastructure.yml | 11 +- hub/Dockerfile | 9 +- hub/docs/deployment.md | 6 +- hub/script/check-image/index.ts | 24 +- infrastructure/.gitignore | 2 + infrastructure/README.md | 24 +- infrastructure/azure/.terraform.lock.hcl | 1 + infrastructure/azure/database.tf | 5 + infrastructure/azure/deploy-hub/index.test.ts | 392 +++++++++++++++++- infrastructure/azure/deploy-hub/index.ts | 135 +++--- infrastructure/azure/deployment-access.tf | 16 + infrastructure/azure/environment.tf | 5 + .../azure/foundation/.terraform.lock.hcl | 22 + infrastructure/azure/foundation/main.tf | 89 ++++ .../foundation/tests/foundation.tftest.hcl | 42 ++ infrastructure/azure/main.tf | 2 +- infrastructure/azure/networking.tf | 1 + infrastructure/azure/outputs.tf | 3 + .../azure/provision/.terraform.lock.hcl | 22 + infrastructure/azure/provision/main.tf | 116 ++++++ .../provision/tests/provision.tftest.hcl | 43 ++ infrastructure/azure/stop-hub/index.test.ts | 224 ++++++++++ infrastructure/azure/stop-hub/index.ts | 52 +++ infrastructure/azure/storage.tf | 2 + .../azure/tests/deployment.tftest.hcl | 114 +++++ .../tests/fixtures/azure-assigned/main.tf | 46 ++ infrastructure/azure/variables.tf | 6 + infrastructure/azure/versions.tf | 2 +- .../installer/command/describe/index.ts | 76 ++++ 29 files changed, 1396 insertions(+), 96 deletions(-) create mode 100644 infrastructure/azure/deployment-access.tf create mode 100644 infrastructure/azure/foundation/.terraform.lock.hcl create mode 100644 infrastructure/azure/foundation/main.tf create mode 100644 infrastructure/azure/foundation/tests/foundation.tftest.hcl create mode 100644 infrastructure/azure/provision/.terraform.lock.hcl create mode 100644 infrastructure/azure/provision/main.tf create mode 100644 infrastructure/azure/provision/tests/provision.tftest.hcl create mode 100644 infrastructure/azure/stop-hub/index.test.ts create mode 100644 infrastructure/azure/stop-hub/index.ts create mode 100644 infrastructure/azure/tests/fixtures/azure-assigned/main.tf create mode 100644 infrastructure/installer/command/describe/index.ts diff --git a/.github/workflows/hub-infrastructure.yml b/.github/workflows/hub-infrastructure.yml index b827faff..6884d382 100644 --- a/.github/workflows/hub-infrastructure.yml +++ b/.github/workflows/hub-infrastructure.yml @@ -14,17 +14,22 @@ defaults: jobs: terraform: - name: Terraform ${{ matrix.command }} + name: Terraform ${{ matrix.version }} ${{ matrix.module }} ${{ matrix.command }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: + version: ["1.13.4", "1.17.1"] + module: [".", "foundation", "provision"] command: ["fmt -check -recursive", "validate", "test"] + defaults: + run: + working-directory: infrastructure/azure/${{ matrix.module }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2 with: - terraform_version: 1.13.4 + terraform_version: ${{ matrix.version }} terraform_wrapper: false - run: terraform init -backend=false -input=false -lockfile=readonly - run: terraform ${{ matrix.command }} @@ -37,4 +42,4 @@ jobs: - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 24 - - run: node --test deploy-hub/index.test.ts + - run: node --test deploy-hub/index.test.ts stop-hub/index.test.ts diff --git a/hub/Dockerfile b/hub/Dockerfile index 4d94bde0..fb17e79a 100644 --- a/hub/Dockerfile +++ b/hub/Dockerfile @@ -6,7 +6,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends openssl ca-cert COPY package.json package-lock.json .npmrc ./ COPY schema ./schema COPY prisma7.config.ts ./ -RUN npm ci --engine-strict +ARG NPM_REGISTRY +RUN if [ -n "$NPM_REGISTRY" ]; then \ + node -e 'const fs=require("node:fs"); const lock=JSON.parse(fs.readFileSync("package-lock.json","utf8")); for(const entry of Object.values(lock.packages)){delete entry.resolved} fs.writeFileSync("package-lock.json",JSON.stringify(lock))'; \ + export npm_config_registry="$NPM_REGISTRY"; \ + fi; \ + npm ci --engine-strict --no-audit --no-fund COPY . . RUN npm run build:runtime @@ -20,7 +25,7 @@ COPY --from=build /app/package.json /app/package-lock.json /app/.npmrc ./ COPY --from=build /app/schema ./schema COPY --from=build /app/prisma7.config.ts ./ COPY --from=build /app/node_modules ./node_modules -RUN npm prune --omit=dev --ignore-scripts && npm cache clean --force +RUN npm prune --omit=dev --ignore-scripts --offline && npm cache clean --force COPY --from=build /app/dist ./dist USER node EXPOSE 3001 diff --git a/hub/docs/deployment.md b/hub/docs/deployment.md index 22b4a1bc..bebb70bc 100644 --- a/hub/docs/deployment.md +++ b/hub/docs/deployment.md @@ -4,7 +4,9 @@ The Hub image runs the frontend, API, and both workers together. It can expose G ## Image and runtime -Build from the repository root with `docker build -t replica-hub:check hub`. The image runs as a non-root user and contains compiled application code and migration tooling. It writes no application state to its writable layer. Publish and deploy the same image digest for setup and Hub. +Build from the repository root with `docker build -t replica-hub:check hub`. When an npm proxy is required, add `--build-arg NPM_REGISTRY=https://YOUR_PROXY/`. This resolves locked dependency versions through that registry without changing the source lockfile; the registry URL must not contain credentials. The image runs as a non-root user and contains compiled application code and migration tooling. It writes no application state to its writable layer. Publish and deploy the same image digest for setup and Hub. + +Host npm configuration is not inherited by the image build. Without `NPM_REGISTRY`, the container needs public npm access. Use `--build-arg NPM_REGISTRY=https://packagefeedproxy.microsoft.io/npm/` where public npm access is not allowed. Explicit scoped registry mappings in the copied `.npmrc` still apply. Publishing is an explicit operator step, not a side effect of CI. From the approved checkout, authenticate to an authorized registry, then run `docker buildx build --platform linux/amd64 --tag YOUR_REGISTRY/replica-hub:YOUR_COMMIT --push hub`. Resolve the published digest with `docker buildx imagetools inspect YOUR_REGISTRY/replica-hub:YOUR_COMMIT` and give deployment the `registry/repository@sha256:...` reference, never the mutable tag. @@ -53,7 +55,7 @@ The deployment orchestrator owns serialization of the complete release, includin ## Scope and evidence -CI checks the image against disposable PostgreSQL, including setup, frontend/API serving, readiness, and termination. This is not a supported local-production installation. The existing devcontainer remains the development environment. +CI checks the image against disposable PostgreSQL, including setup, frontend/API serving, readiness, and termination. Set `NPM_REGISTRY` when running `node hub/script/check-image/index.ts` locally if the build requires a proxy. This is not a supported local-production installation. The existing devcontainer remains the development environment. The [Azure reference installation](../../infrastructure/README.md) uses Terraform and a serialized maintenance-window release. It consumes the same image but does not yet configure the optional authentication settings. Add those protected settings and the exact Container Apps proxy-chain length before enabling authentication there. diff --git a/hub/script/check-image/index.ts b/hub/script/check-image/index.ts index ce0b8fe4..037e5585 100644 --- a/hub/script/check-image/index.ts +++ b/hub/script/check-image/index.ts @@ -31,9 +31,27 @@ export async function checkImage() { let applicationCreated = false let imageCreated = false try { - execFileSync("docker", ["build", "-t", image, resolve(import.meta.dirname, "../..")], { - stdio: "inherit", - }) + const registry = process.env.NPM_REGISTRY + if (registry) { + const url = new URL(registry) + assert( + url.protocol === "https:" && !url.username && !url.password && !url.search && !url.hash, + "NPM_REGISTRY must be an HTTPS URL without credentials, query, or fragment", + ) + } + execFileSync( + "docker", + [ + "build", + "-t", + image, + ...(registry ? ["--build-arg", `NPM_REGISTRY=${registry}`] : []), + resolve(import.meta.dirname, "../.."), + ], + { + stdio: "inherit", + }, + ) imageCreated = true docker("network", "create", network) networkCreated = true diff --git a/infrastructure/.gitignore b/infrastructure/.gitignore index 5e77b516..33f87571 100644 --- a/infrastructure/.gitignore +++ b/infrastructure/.gitignore @@ -1,4 +1,6 @@ .tools/ +node_modules/ +.local/ **/.terraform/ **/*.tfstate* **/*.tfplan diff --git a/infrastructure/README.md b/infrastructure/README.md index 3fc58f1a..8ab50651 100644 --- a/infrastructure/README.md +++ b/infrastructure/README.md @@ -1,10 +1,10 @@ # Azure Hub installation -Operators can provision a durable Hub, run setup before every release, and replace the image without replacing its database or artifacts. This reference chooses Terraform with AzureRM to follow the team's Terraform experience; it is an implementation choice, not an application dependency. The configuration is flat under [`azure/`](azure/) rather than split into one-resource modules. +Operators can provision a durable Hub, run setup before every release, and replace the image without replacing its database or artifacts. This reference chooses Terraform with AzureRM to follow the team's Terraform experience; it is an implementation choice, not an application dependency. The main installation is flat under [`azure/`](azure/) rather than split into one-resource modules. Definitions are grouped by responsibility: [`networking.tf`](azure/networking.tf), [`database.tf`](azure/database.tf), [`storage.tf`](azure/storage.tf), [`roles.tf`](azure/roles.tf), [`environment.tf`](azure/environment.tf), [`application.tf`](azure/application.tf), and [`setup.tf`](azure/setup.tf). Shared tags and the resource group remain in [`main.tf`](azure/main.tf). These files form one Terraform module; moving declarations between them does not change resource addresses or require state migration. -This installs the current Hub, not hosted previews, review collaboration, user admission, or a sign-in provider. A public HTTPS endpoint does not authorize users or isolate prototype previews. Confirm product access requirements before exposing sensitive data. +The current Hub supports GitHub sign-in and reusable administrator enrollment when configured. This reference does not configure those optional authentication settings by default, and hosted previews and review collaboration are not built yet. Supply the protected settings from [Hub deployment](../hub/docs/deployment.md#image-and-runtime), including the exact protected proxy-chain length, before enabling authentication. A public HTTPS endpoint does not authorize users or isolate prototype previews. ## What is created @@ -16,10 +16,18 @@ Each installation has its own resource group tagged with its application, instal - Separate Hub and setup user-assigned identities. Only Hub receives `Storage Blob Data Contributor` on the artifacts container. Each identity can read only its referenced Key Vault secrets and, when configured, pull from an existing ACR. No vault or registry is created or owned. - Log Analytics with 30-day retention, a Container Apps environment, a manual setup job, and, after successful setup, one combined Hub app. +Two optional modules have separate Terraform states and lifecycles. [`azure/foundation/`](azure/foundation/main.tf) creates an installation-owned ACR and RBAC-enabled Key Vault in a separate resource group, with `AcrPush` and `Key Vault Secrets Officer` grants for the deployment identity. The vault has 90-day soft deletion and purge protection. Pass its registry and vault outputs to the main installation; the main module still accepts existing resources and never owns their teardown. + +[`azure/provision/`](azure/provision/main.tf) creates a temporary private-network database-role provisioning job and identity in the installation's Container Apps environment. It accepts the same immutable image and exactly three versioned Key Vault references: `DATABASE_URL` for the administrator, `HUB_MIGRATION_PASSWORD`, and `HUB_RUNTIME_PASSWORD`. The job has one replica, no automatic retries, and a ten-minute timeout. An authorized operator must run it once against a fresh database, verify that exact execution succeeded, and remove the temporary job, identity, and grants before release. Do not rerun it against an installed database or grant its administrator secret to Hub or setup. The existing private-network operator script remains an alternative. + +Initialize each optional module against its own protected backend state key, review its declared inputs and Terraform plan, and authorize its apply separately. Never share a state key with the main installation. Neither module starts Hub or replaces the serialized release command. + The app has external HTTPS ingress and minimum and maximum one replica **per revision**. Multiple-revision mode permits explicit deactivation before releases. Azure's single-revision mode deliberately overlaps old and new processes while readiness is checked, so it is unsuitable for this maintenance-window strategy. Do not activate old revisions alongside the combined worker. This is not a high-availability or zero-downtime installation. The runtime receives `RUNTIME_MODE=combined`, `HOST=0.0.0.0`, `PORT=3001`, `CLIENT_DIRECTORY=/app/dist/client`, `SHUTDOWN_TIMEOUT_MS=30000`, and `NODE_ENV=production`. Artifacts use `ARTIFACT_STORAGE_PROVIDER=azure`, the provisioned `AZURE_STORAGE_ACCOUNT`, `AZURE_STORAGE_CONTAINER=artifacts`, and Hub's `AZURE_CLIENT_ID`. `DATABASE_URL` references a Key Vault secret for the `replica_runtime` database role with `sslmode=verify-full`. Optional `runtime_secret_references` map environment names to vault resource IDs and HTTPS secret URLs. Container Apps resolves these references using Hub's identity; Hub does not implement a separate secret-resolution framework. +Terraform ignores only Azure-assigned PostgreSQL availability-zone and Container Apps infrastructure-resource-group values. It still checks other drift and owns both images. The installation output exposes authoritative app, resource-group, and storage-account IDs; before the first release, the app ID is `null`. + The setup job uses the same digest and command `node dist/server/scripts/setup/index.js`. Its only environment settings are `NODE_ENV=production`, `RUNTIME_MODE=setup`, `DATABASE_RUNTIME_ROLE=replica_runtime`, and its own Key Vault-backed `DATABASE_URL` for `replica_migration`. It receives no artifact configuration or additional runtime secrets. It has one replica, no automatic retries, and a 30-minute timeout. Hub's readiness probe is `/api/ready`; startup and liveness use `/api/health`. Azure allows 60 seconds for termination, covering the 30-second application drain and up to five seconds of telemetry flushing with platform margin. Neither workload receives the PostgreSQL administrator credential. A separate, one-time operator script provisions the migration and runtime roles over the private database network. Setup owns schema changes and refreshes the runtime's data permissions after migrations. See [database access](../hub/docs/database.md#deployment-database-roles). Managed identity is used for Blob, Key Vault, and ACR, not database authentication. @@ -28,7 +36,7 @@ Neither workload receives the PostgreSQL administrator credential. A separate, o Use the Terraform version constrained in [`versions.tf`](azure/versions.tf), a Node.js version supported by `scripts/package.json`, and a current Azure CLI with Container Apps job and revision commands. CI performs formatting, validation, mocked Terraform tests, and release-orchestration tests without cloud credentials. It does not deploy. -An authorized administrator must first create the **separate** Terraform backend: a dedicated state resource group, Storage account, and private container. Enable TLS, Entra authorization, Blob versioning and soft deletion; restrict network access to deployment runners and grant only the deployment identity `Storage Blob Data Contributor` on the state container. These bootstrap resources are not part of this installation's state or teardown. Never store this state in the app's artifacts account. +An authorized administrator must first create the **separate** Terraform backend: a dedicated state resource group, Storage account, and private container. Enable TLS, Entra authorization, Blob versioning and soft deletion; disable shared-key authorization and anonymous Blob access, and grant the deployment identity `Storage Blob Data Contributor` on the state container. Apply any network restrictions required by deployment policy and ensure the runner can reach the account. These bootstrap resources are not part of this installation's state or teardown. Never store this state in the app's artifacts account. For automation, configure a dedicated deployment identity with a federated workload identity credential rather than a client secret. Bind its issuer to `https://token.actions.githubusercontent.com`, audience to `api://AzureADTokenExchange`, and subject to the exact protected repository/environment, such as `repo:YOUR_ORG/YOUR_REPO:environment:hub-production`. Protect that environment and deployment branch. A future release workflow needs `id-token: write` and must authenticate Azure CLI as the same identity; the infrastructure-check workflow intentionally has no such permission. @@ -36,6 +44,8 @@ Set `ARM_CLIENT_ID`, `ARM_TENANT_ID`, `ARM_SUBSCRIPTION_ID`, and `ARM_USE_OIDC=t The deployment identity needs resource creation/update rights for the installation, role-assignment rights for the artifact container, referenced Key Vault secrets and optional registry, and Blob data-plane rights on the installation's account for the lock blob. Resource `Contributor` alone does not grant data access or role assignment. An administrator must authorize initial resource-group creation and inherited Blob data access for bootstrap, then narrow ongoing deployment grants to the installation resource group, referenced secrets, existing registry, and backend container. Precreated/imported infrastructure needs a separately reviewed initial plan rather than the empty-state bootstrap command. Do not grant either workload identity deployment or state access. +The main module's optional `grant_deployment_storage_access=true` grants the authenticated Terraform deployment identity `Storage Blob Data Contributor` on the installation account. It defaults to `false`; use it only when this account-level grant matches the deployment policy. It does not grant access to the separate state backend. + Register `Microsoft.App`, `Microsoft.OperationalInsights`, `Microsoft.Network`, `Microsoft.Storage`, `Microsoft.DBforPostgreSQL`, and `Microsoft.ManagedIdentity` beforehand. The provider does not register services across the subscription. Allow time for newly assigned RBAC roles to propagate before setup. Copy [`backend.hcl.example`](azure/backend.hcl.example) to `infrastructure/azure/backend.hcl` and [`installation.tfvars.example`](azure/installation.tfvars.example) to `infrastructure/azure/installation.tfvars`. Both real files are ignored. Replace every placeholder, choose a globally unique name, and resolve an existing image to its real `registry/repository@sha256:...` digest. Use either a public image or an existing ACR; private non-ACR registry credentials and ACR's ABAC repository-permissions mode are not covered by this reference. @@ -84,7 +94,7 @@ The operator must approve the maintenance window and inspect intended configurat Terraform owns the image in both resources. There is deliberately **no** `ignore_changes` or out-of-band `az containerapp update --image`: a later infrastructure apply cannot silently use a stale image. Provisioning and rollout are separated by the two phases above. Never run a full Terraform apply directly on an installed Hub, start a setup job manually, or change revisions in the portal outside this serialized procedure. Terraform state locking alone does not protect setup-to-rollout ordering. The app's `prevent_destroy` guard rejects accidental `deploy_hub=false` on later full plans. -Plans reject resource deletion and replacement during normal releases. A database major upgrade, region move, rename, or replacement needs a reviewed migration/recovery plan, not an override of that guard. The release script deletes its sensitive saved plan on exit. It leaves Hub stopped and retains the infinite lease after any release failure; it never reactivates an older process automatically against a possibly migrated schema. +Plans reject resource deletion and replacement during normal releases. A database major upgrade, region move, rename, or replacement needs a reviewed migration/recovery plan, not an override of that guard. The release script stores sensitive plans under `TF_DATA_DIR` when set, otherwise under the module's `.terraform/`, and deletes them on exit. Failures identify a fixed command label and available exit code without printing arguments or tool output. It leaves Hub stopped and retains the infinite lease after any release failure; it never reactivates an older process automatically against a possibly migrated schema. ### Recover a failed release @@ -139,16 +149,18 @@ Storage-account deletion does not preserve a usable Blob backup just because obj ## Credential-free checks and evidence limits -From `infrastructure/azure/`: +From `infrastructure/azure/`, check the main installation: ```sh terraform init -backend=false -input=false -lockfile=readonly terraform fmt -check -recursive terraform validate terraform test -node --test deploy-hub/index.test.ts +node --test deploy-hub/index.test.ts stop-hub/index.test.ts ``` +Run the same Terraform initialization, formatting, validation, and test commands from `foundation/` and `provision/` to check the optional modules. CI checks each module using both supported Terraform versions in [the workflow matrix](../.github/workflows/hub-infrastructure.yml). + Terraform tests use AzureRM mocks, not an Azure subscription. Orchestration tests replace command execution and HTTPS requests, proving ordering and failure gates rather than Azure behavior. Readiness includes a bounded, non-writing container-properties request through Hub's managed identity; it detects unavailable storage but does not prove write or delete authorization. Worker readiness remains initialization-only. Before production, an authorized tenant exercise must verify provisioning, private PostgreSQL connectivity and TLS, separate database roles, Key Vault resolution, identity-based Blob/ACR access after RBAC propagation, failed setup blocking rollout, maintenance shutdown and revision behavior, HTTPS/readiness, persistence across replacement, custom TLS when configured, backup restoration, and owned-resource teardown. No real-cloud result is implied by these local checks. The release command and its tests are TypeScript running directly in Node.js. Terraform's own tests use `.tftest.hcl`; neither requires Python. From the repository root, `npm run typecheck --prefix scripts` checks the TypeScript operational scripts as well. diff --git a/infrastructure/azure/.terraform.lock.hcl b/infrastructure/azure/.terraform.lock.hcl index 30c89942..0bd5c529 100644 --- a/infrastructure/azure/.terraform.lock.hcl +++ b/infrastructure/azure/.terraform.lock.hcl @@ -5,6 +5,7 @@ provider "registry.terraform.io/hashicorp/azurerm" { version = "4.50.0" constraints = "4.50.0" hashes = [ + "h1:Q/JIe7B0F7cziHSDBE7Yr+mFxhtNCDe3JjxuobOGh6w=", "h1:j2bM6QYxBw4dKDlNLR2lAWP6Dggv5lhyGU5VREOfaDw=", "zh:11899f771a9f03c4e5d222d73ce842e930d26af8aaba88b99a2f0638265331dc", "zh:2450a4258f40f6ba1f6705a10b2bdb6ff2067ab4c2a6938135391df03c974251", diff --git a/infrastructure/azure/database.tf b/infrastructure/azure/database.tf index 4775994b..a6e21863 100644 --- a/infrastructure/azure/database.tf +++ b/infrastructure/azure/database.tf @@ -14,6 +14,11 @@ resource "azurerm_postgresql_flexible_server" "database" { private_dns_zone_id = azurerm_private_dns_zone.database.id tags = local.tags depends_on = [azurerm_private_dns_zone_virtual_network_link.database] + + lifecycle { + # Azure selects the zone because this module does not pin one. + ignore_changes = [zone] + } } resource "azurerm_postgresql_flexible_server_configuration" "tls" { diff --git a/infrastructure/azure/deploy-hub/index.test.ts b/infrastructure/azure/deploy-hub/index.test.ts index d3206551..b0a0a70e 100644 --- a/infrastructure/azure/deploy-hub/index.test.ts +++ b/infrastructure/azure/deploy-hub/index.test.ts @@ -2,7 +2,9 @@ import assert from "node:assert/strict" import childProcess from "node:child_process" import { afterEach, describe, it, mock } from "node:test" import timers from "node:timers/promises" +import { inspect } from "node:util" +import { describeCommand } from "../../installer/command/describe/index.ts" import { deployHub } from "./index.ts" afterEach(() => { @@ -14,11 +16,26 @@ function installationScenario({ setupStatus = "Succeeded", lockAvailable = true, releaseError, + appResponse = { + properties: { + latestRevisionName: "test-hub--latest", + latestReadyRevisionName: "test-hub--latest", + }, + }, + revisionResponse = { name: "test-hub--latest", properties: { active: true } }, + httpStatuses = [200], + commandFailure, + commandOutput, }: { appCreated?: boolean setupStatus?: string lockAvailable?: boolean releaseError?: Error + appResponse?: unknown + revisionResponse?: unknown + httpStatuses?: number[] + commandFailure?: { args: string[]; error: Error; occurrence?: number } + commandOutput?: { args: string[]; stdout: string } } = {}) { const installation = { name: "test-hub", @@ -36,6 +53,7 @@ function installationScenario({ const events: string[][] = [] const commandOptions: childProcess.ExecFileSyncOptions[] = [] const replicaReads = new Map() + let matchingCommands = 0 let targeted = false function execute( file: string, @@ -45,6 +63,12 @@ function installationScenario({ const command: string[] = [file, ...args] events.push(command) commandOptions.push(options) + if (commandFailure?.args.every((argument, index) => args[index] === argument)) { + matchingCommands += 1 + if (matchingCommands === (commandFailure.occurrence ?? 1)) throw commandFailure.error + } + if (file === "az" && commandOutput?.args.every((argument, index) => args[index] === argument)) + return commandOutput.stdout if (file === "terraform") { switch (args[1]) { case "output": @@ -93,7 +117,7 @@ function installationScenario({ return JSON.stringify("lease-id") } if (args.includes("release") && releaseError) throw releaseError - return "null" + return "" case "containerapp revision list": return JSON.stringify([ { name: "old-active", properties: { active: true } }, @@ -111,24 +135,24 @@ function installationScenario({ return JSON.stringify({ name: "setup-execution" }) case "containerapp revision deactivate": case "containerapp revision activate": - return "null" + return "" + case "containerapp revision show": + return JSON.stringify(revisionResponse) default: if (args.slice(0, 2).join(" ") === "containerapp show") { - return JSON.stringify({ - properties: { - latestRevisionName: "new", - latestReadyRevisionName: "new", - }, - }) + return JSON.stringify(appResponse) } throw new Error(`Unexpected Azure command: ${command.join(" ")}`) } } mock.method(childProcess, "execFileSync", execute) mock.method(timers, "setTimeout", async () => undefined) + let readinessRequests = 0 mock.method(globalThis, "fetch", async (url: Parameters[0]) => { events.push(["HTTPS", String(url)]) - return new Response(null, { status: 200 }) + const status = httpStatuses[Math.min(readinessRequests, httpStatuses.length - 1)] + readinessRequests += 1 + return new Response(null, { status }) }) const args = [ "release", @@ -148,6 +172,9 @@ function installationScenario({ describe("deployHub()", () => { it("drains all old revisions and completes setup before rollout, readiness, and lease release", async () => { const { args, events, replicaReads, commandOptions } = installationScenario() + const output: unknown[][] = [] + mock.method(console, "error", (...args: unknown[]) => output.push(args)) + const completion = mock.method(console, "log", () => events.push(["complete"])) await deployHub(args) @@ -181,13 +208,30 @@ describe("deployHub()", () => { (event, index) => index > rollout && event[0] === "terraform" && event[2] === "apply", ) assert.ok(rolloutApplied > rollout) + const inspected = events.findIndex( + (event) => event.slice(0, 4).join(" ") === "az containerapp revision show", + ) + assert.ok(inspected > rolloutApplied) + assert.equal(events[inspected][events[inspected].indexOf("--revision") + 1], "test-hub--latest") + assert.ok(!events.some((event) => event.includes("activate"))) const ready = events.findIndex((event) => event[0] === "HTTPS") const release = events.findIndex( (event) => event.slice(0, 5).join(" ") === "az storage blob lease release", ) assert.deepEqual(events[ready], ["HTTPS", "https://hub.example.com/api/ready"]) - assert.ok(ready > rolloutApplied) + assert.ok(ready > inspected) + assert.ok( + events.some( + (event, index) => + index > inspected && + index < ready && + event.slice(0, 3).join(" ") === "az containerapp show", + ), + ) assert.ok(release > ready) + assert.equal(completion.mock.callCount(), 1) + assert.ok(events.findIndex((event) => event[0] === "complete") > release) + assert.doesNotMatch(JSON.stringify(output), /lease is retained/) assert.ok( commandOptions.every( ({ stdio }) => @@ -210,8 +254,9 @@ describe("deployHub()", () => { assert.ok(!events.some((event) => event[0] === "HTTPS")) }) - it("runs setup before creating the first Hub app", async () => { - const { args, events } = installationScenario({ appCreated: false }) + it("runs setup before the first Hub app and verifies its already-active revision without activating it again", async () => { + const { args, events } = installationScenario({ appCreated: false, httpStatuses: [503, 200] }) + mock.method(console, "warn", () => undefined) await deployHub(args) @@ -226,8 +271,232 @@ describe("deployHub()", () => { ) assert.ok(setup > events.indexOf(plans[0])) assert.ok(events.indexOf(plans[1]) > setup) + const inspected = events.filter( + (event) => event.slice(0, 4).join(" ") === "az containerapp revision show", + ) + assert.equal(inspected.length, 1) + assert.equal(inspected[0][inspected[0].indexOf("--revision") + 1], "test-hub--latest") + assert.ok(!events.some((event) => event.includes("activate"))) + const ready = events.findIndex((event) => event[0] === "HTTPS") + const readinessRequests = events.filter((event) => event[0] === "HTTPS") + const release = events.findIndex( + (event) => event.slice(0, 5).join(" ") === "az storage blob lease release", + ) + assert.ok(ready > events.indexOf(inspected[0])) + assert.equal(readinessRequests.length, 2) + assert.ok(release > events.indexOf(readinessRequests[1])) }) + it("activates an inactive latest revision only after successful setup and the full rollout apply", async () => { + const { args, events } = installationScenario({ + revisionResponse: { name: "test-hub--latest", properties: { active: false } }, + }) + + await deployHub(args) + + const setupSucceeded = events.findIndex( + (event) => event.slice(0, 5).join(" ") === "az containerapp job execution show", + ) + const rollout = events.findIndex( + (event) => + event[0] === "terraform" && + event[2] === "plan" && + !event.some((argument) => argument.startsWith("-target=")), + ) + const applied = events.findIndex( + (event, index) => index > rollout && event[0] === "terraform" && event[2] === "apply", + ) + const inspections = events.filter( + (event) => event.slice(0, 4).join(" ") === "az containerapp revision show", + ) + const activations = events.filter( + (event) => event.slice(0, 4).join(" ") === "az containerapp revision activate", + ) + assert.equal(inspections.length, 1) + assert.equal(activations.length, 1) + for (const event of [...inspections, ...activations]) { + assert.equal(event[event.indexOf("--revision") + 1], "test-hub--latest") + } + assert.ok(setupSucceeded >= 0) + assert.ok(rollout > setupSucceeded) + assert.ok(applied > rollout) + assert.ok(events.indexOf(inspections[0]) > applied) + assert.ok(events.indexOf(activations[0]) > events.indexOf(inspections[0])) + const ready = events.findIndex((event) => event[0] === "HTTPS") + const released = events.findIndex( + (event) => event.slice(0, 5).join(" ") === "az storage blob lease release", + ) + assert.ok(ready > events.indexOf(activations[0])) + assert.ok(released > ready) + }) + + for (const { name, scenario, inspectRevision } of [ + { + name: "missing latest revision name", + scenario: { appResponse: { properties: {} } }, + inspectRevision: false, + }, + { + name: "non-string latest revision name", + scenario: { + appResponse: { properties: { latestRevisionName: 17, latestReadyRevisionName: 17 } }, + }, + inspectRevision: false, + }, + { + name: "empty latest revision name", + scenario: { + appResponse: { properties: { latestRevisionName: "", latestReadyRevisionName: "" } }, + }, + inspectRevision: false, + }, + { + name: "whitespace-only latest revision name", + scenario: { + appResponse: { properties: { latestRevisionName: " ", latestReadyRevisionName: " " } }, + revisionResponse: { name: " ", properties: { active: false } }, + }, + inspectRevision: false, + }, + { + name: "latest revision name containing whitespace", + scenario: { + appResponse: { + properties: { + latestRevisionName: "test-hub--latest\nprivate-value", + latestReadyRevisionName: "test-hub--latest\nprivate-value", + }, + }, + revisionResponse: { + name: "test-hub--latest\nprivate-value", + properties: { active: false }, + }, + }, + inspectRevision: false, + }, + { + name: "mismatched inactive revision response", + scenario: { revisionResponse: { name: "old-active", properties: { active: false } } }, + inspectRevision: true, + }, + { + name: "missing activation state", + scenario: { revisionResponse: { name: "test-hub--latest", properties: {} } }, + inspectRevision: true, + }, + { + name: "non-boolean activation state", + scenario: { + revisionResponse: { name: "test-hub--latest", properties: { active: "false" } }, + }, + inspectRevision: true, + }, + { + name: "absent revision response", + scenario: { revisionResponse: null }, + inspectRevision: true, + }, + ]) { + it(`retains the lease without activation or readiness when Azure returns ${name}`, async () => { + const { args, events } = installationScenario(scenario) + const output: unknown[][] = [] + mock.method(console, "error", (...args: unknown[]) => output.push(args)) + const completion = mock.method(console, "log", (...args: unknown[]) => output.push(args)) + + await assert.rejects(deployHub(args), (error: unknown) => { + assert.ok(error instanceof Error) + assert.equal(error.cause, undefined) + const rendered = inspect([error, output], { depth: 10, showHidden: true }) + assert.ok(!rendered.includes("test-hub")) + assert.ok(!rendered.includes("private-value")) + return true + }) + + const revisionInspections = events.filter( + (event) => event.slice(0, 4).join(" ") === "az containerapp revision show", + ) + assert.equal(revisionInspections.length, inspectRevision ? 1 : 0) + assert.ok(!events.some((event) => event.includes("activate"))) + assert.ok(!events.some((event) => event[0] === "HTTPS")) + assert.ok(!events.some((event) => event.includes("release"))) + assert.match(JSON.stringify(output), /lease is retained/) + assert.equal(completion.mock.callCount(), 0) + }) + } + + for (const failure of [ + { name: "revision inspection", args: ["containerapp", "revision", "show"], active: true }, + { name: "revision activation", args: ["containerapp", "revision", "activate"], active: false }, + { + name: "Azure readiness inspection of an active revision", + args: ["containerapp", "show"], + occurrence: 2, + active: true, + }, + ]) { + it(`reports safe operation and status and stops after a genuine ${failure.name} failure`, async () => { + const privateOutput = "synthetic-private-child-output" + const childError = Object.assign(new Error(privateOutput), { + status: 17, + stdout: Buffer.from(privateOutput), + stderr: Buffer.from(privateOutput), + cause: new Error(privateOutput), + }) + const { args, events } = installationScenario({ + revisionResponse: { name: "test-hub--latest", properties: { active: failure.active } }, + commandFailure: { ...failure, error: childError }, + }) + const output: unknown[][] = [] + mock.method(console, "error", (...args: unknown[]) => output.push(args)) + const completion = mock.method(console, "log", (...args: unknown[]) => output.push(args)) + + await assert.rejects(deployHub(args), (error: unknown) => { + assert.ok(error instanceof Error) + assert.ok(error.message.includes(describeCommand("az", failure.args).label)) + assert.match(error.message, /\bexit(?:ed with)?(?: code| status)?[: ]+17\b/i) + assert.equal(error.cause, undefined) + const rendered = inspect([error, output], { depth: 10, showHidden: true }) + for (const value of [ + privateOutput, + "lease-id", + "test-subscription", + "test-hub", + "hub.example.com", + ]) { + assert.ok(!rendered.includes(value)) + } + return true + }) + + assert.ok( + events.some((event) => event.slice(0, 5).join(" ") === "az storage blob lease acquire"), + ) + assert.equal( + events.filter((event) => event.slice(0, 4).join(" ") === "az containerapp job start") + .length, + 1, + ) + assert.equal( + events.filter((event) => event[0] === "terraform" && event[2] === "apply").length, + 2, + ) + assert.equal( + events.filter((event) => + failure.args.every((argument, index) => event[index + 1] === argument), + ).length, + failure.occurrence ?? 1, + ) + assert.equal( + events.filter((event) => event.includes("activate")).length, + failure.active ? 0 : 1, + ) + assert.ok(!events.some((event) => event[0] === "HTTPS")) + assert.ok(!events.some((event) => event.includes("release"))) + assert.match(JSON.stringify(output), /lease is retained/) + assert.equal(completion.mock.callCount(), 0) + }) + } + it("does not stop Hub or run setup while another release holds the lease", async () => { const { args, events } = installationScenario({ lockAvailable: false }) @@ -241,10 +510,71 @@ describe("deployHub()", () => { assert.ok(!events.some((event) => event[0] === "terraform" && event[2] === "apply")) }) + for (const response of [ + { + name: "empty lease acquisition output", + args: ["storage", "blob", "lease", "acquire"], + stdout: "", + retainsLease: false, + }, + { + name: "malformed job start output", + args: ["containerapp", "job", "start"], + stdout: + 'synthetic-private-output {"credential":"synthetic-private-credential","lease":"lease-id"}', + retainsLease: true, + }, + ]) { + it(`rejects ${response.name} with a safe operation label and no deployment completion`, async () => { + const { args, events } = installationScenario({ commandOutput: response }) + const output: unknown[][] = [] + mock.method(console, "error", (...args: unknown[]) => output.push(args)) + const completion = mock.method(console, "log", (...args: unknown[]) => output.push(args)) + + await assert.rejects(deployHub(args), (error: unknown) => { + assert.ok(error instanceof Error) + assert.ok(error.message.includes(describeCommand("az", response.args).label)) + assert.equal(error.cause, undefined) + const rendered = inspect([error, output], { depth: 10, showHidden: true }) + for (const value of [ + "synthetic-private-output", + "synthetic-private-credential", + "lease-id", + "test-subscription", + "test-hub", + "hub.example.com", + ]) + assert.ok(!rendered.includes(value)) + return true + }) + + assert.ok( + events.some((event) => + response.args.every((argument, index) => event[index + 1] === argument), + ), + ) + assert.equal( + events.filter((event) => event.slice(0, 4).join(" ") === "az containerapp job start") + .length, + response.retainsLease ? 1 : 0, + ) + assert.equal( + events.filter((event) => event[0] === "terraform" && event[2] === "apply").length, + response.retainsLease ? 1 : 0, + ) + assert.equal(JSON.stringify(output).includes("lease is retained"), response.retainsLease) + assert.ok(!events.some((event) => event.includes("activate"))) + assert.ok(!events.some((event) => event[0] === "HTTPS")) + assert.ok(!events.some((event) => event.includes("release"))) + assert.equal(completion.mock.callCount(), 0) + }) + } + it("redacts child diagnostics and retains the lock warning when lease release fails", async () => { const leaseId = "lease-id" const privateOutput = `private-credential ${leaseId}` const releaseError = Object.assign(new Error(`az --lease-id ${privateOutput}`), { + status: 23, stdout: Buffer.from(privateOutput), stderr: Buffer.from(privateOutput), cause: new Error(privateOutput), @@ -252,13 +582,18 @@ describe("deployHub()", () => { const { args, events } = installationScenario({ releaseError }) const output: unknown[][] = [] mock.method(console, "error", (...args: unknown[]) => output.push(args)) - mock.method(console, "log", (...args: unknown[]) => output.push(args)) + const completion = mock.method(console, "log", (...args: unknown[]) => output.push(args)) await assert.rejects(deployHub(args), (error: unknown) => { assert.ok(error instanceof Error) - assert.match(error.message, /Azure command failed/) + assert.ok( + error.message.includes( + describeCommand("az", ["storage", "blob", "lease", "release"]).label, + ), + ) + assert.match(error.message, /\bexit(?:ed with)?(?: code| status)?[: ]+23\b/i) assert.equal(error.cause, undefined) - const rendered = `${error.stack} ${JSON.stringify(error)} ${JSON.stringify(output)}` + const rendered = inspect([error, output], { depth: 10, showHidden: true }) assert.ok(!rendered.includes(leaseId)) assert.ok(!rendered.includes("private-credential")) return true @@ -267,7 +602,7 @@ describe("deployHub()", () => { assert.ok(events.some((event) => event[0] === "HTTPS")) assert.ok(events.some((event) => event.includes("--lease-id") && event.includes(leaseId))) assert.match(JSON.stringify(output), /lease is retained/) - assert.ok(!JSON.stringify(output).includes("maintenance window is complete")) + assert.equal(completion.mock.callCount(), 0) }) it("refuses bootstrap when Terraform state already contains resources", async () => { @@ -283,4 +618,29 @@ describe("deployHub()", () => { ) assert.ok(!events.some((event) => event[0] === "terraform" && event[2] === "apply")) }) + + it("completes confirmed teardown after a successful blob deletion with empty output", async () => { + const { args, events } = installationScenario({ + appCreated: false, + commandOutput: { args: ["storage", "blob", "delete"], stdout: "" }, + }) + args[0] = "destroy" + args.push("--confirm", "test-hub") + mock.method(console, "log", () => events.push(["complete"])) + + await deployHub(args) + + const plan = events.findIndex((event) => event[0] === "terraform" && event[2] === "plan") + const deleted = events.findIndex( + (event) => event.slice(0, 4).join(" ") === "az storage blob delete", + ) + const applied = events.findIndex((event) => event[0] === "terraform" && event[2] === "apply") + assert.ok(plan >= 0 && events[plan].includes("-destroy")) + assert.ok(deleted > plan) + assert.ok(applied > deleted) + assert.ok(events.findIndex((event) => event[0] === "complete") > applied) + assert.ok(!events.some((event) => event.includes("release") || event.includes("activate"))) + assert.ok(!events.some((event) => event.slice(0, 4).join(" ") === "az containerapp job start")) + assert.ok(!events.some((event) => event[0] === "HTTPS")) + }) }) diff --git a/infrastructure/azure/deploy-hub/index.ts b/infrastructure/azure/deploy-hub/index.ts index 82eb168a..ce800a4a 100644 --- a/infrastructure/azure/deploy-hub/index.ts +++ b/infrastructure/azure/deploy-hub/index.ts @@ -1,11 +1,13 @@ import childProcess from "node:child_process" import { randomUUID } from "node:crypto" -import { rm } from "node:fs/promises" +import { mkdir, rm } from "node:fs/promises" import { resolve } from "node:path" import timers from "node:timers/promises" import { pathToFileURL } from "node:url" import { parseArgs } from "node:util" +import { describeCommand } from "../../installer/command/describe/index.ts" +import { stopHub } from "../stop-hub/index.ts" import type { ContainerApp, DeploymentPlan, Installation, JobExecution, Revision } from "./types.ts" export async function deployHub(args: string[] = process.argv.slice(2)): Promise { @@ -52,7 +54,8 @@ export async function deployHub(args: string[] = process.argv.slice(2)): Promise const subscriptionId = subscription const variablesPath = resolve(variables) const root = resolve(import.meta.dirname, "..") - const planPath = resolve(root, ".terraform", `deployment-${randomUUID()}.tfplan`) + const planDirectory = resolve(process.env.TF_DATA_DIR ?? resolve(root, ".terraform")) + const planPath = resolve(planDirectory, `deployment-${randomUUID()}.tfplan`) const previousMask = process.umask(0o077) let lease: string | undefined @@ -64,37 +67,48 @@ export async function deployHub(args: string[] = process.argv.slice(2)): Promise maxBuffer: 32 * 1024 * 1024, }) return capture ? (output?.trim() ?? "") : "" - } catch { + } catch (error) { // Child errors and output can contain credentials and release lease arguments. + const exitCode = + error instanceof Error && + "status" in error && + typeof error.status === "number" && + Number.isInteger(error.status) && + error.status > 0 + ? error.status + : undefined + const tool = executable === "terraform" ? "Terraform" : "Azure" throw new Error( - executable === "terraform" - ? "Terraform command failed; inspect the installation through authorized tooling." - : "Azure command failed; inspect the installation through authorized tooling.", + `${tool} command failed: ${describeCommand(executable, arguments_).label}${exitCode === undefined ? "" : ` (exit code ${exitCode})`}. Tool output remains private; inspect the installation through authorized tooling.`, ) } } function terraform(arguments_: string[], capture = false): string { return command("terraform", [`-chdir=${root}`, ...arguments_], capture) } - function parseOutput(output: string): T { + function parseOutput(output: string, operation: string): T { try { return JSON.parse(output) as T } catch { - throw new Error("Deployment tooling returned an invalid response.") + throw new Error( + `${operation} returned invalid JSON. Tool output remains private; inspect the installation through authorized tooling.`, + ) } } - function azure(...arguments_: string[]): T { - return parseOutput( - command( - "az", - [...arguments_, "--subscription", subscriptionId, "--only-show-errors", "--output", "json"], - true, - ), + function azureCommand(...arguments_: string[]): string { + return command( + "az", + [...arguments_, "--subscription", subscriptionId, "--only-show-errors", "--output", "json"], + true, ) } + function azure(...arguments_: string[]): T { + return parseOutput(azureCommand(...arguments_), describeCommand("az", arguments_).label) + } function readInstallation(): Installation { const installation = parseOutput( terraform(["output", "-json", "installation"], true), + describeCommand("terraform", ["output", "-json", "installation"]).label, ) if (installation.subscription_id !== subscription) { throw new Error("The selected subscription does not match this Terraform state.") @@ -154,7 +168,10 @@ export async function deployHub(args: string[] = process.argv.slice(2)): Promise if (target) arguments_.push(`-target=${target}`) if (destroy) arguments_.push("-destroy") terraform(arguments_) - const planned = parseOutput(terraform(["show", "-json", planPath], true)) + const planned = parseOutput( + terraform(["show", "-json", planPath], true), + describeCommand("terraform", ["show", "-json", planPath]).label, + ) if (!destroy) { if (planned.resource_changes?.some(({ change }) => change.actions.includes("delete"))) { throw new Error( @@ -172,46 +189,6 @@ export async function deployHub(args: string[] = process.argv.slice(2)): Promise function apply(): void { terraform(["apply", "-input=false", "-lock-timeout=60s", planPath]) } - async function stopHub(installation: Installation): Promise { - if (!installation.app_created) return - const revisions = azure( - "containerapp", - "revision", - "list", - ...appArguments(installation), - ) - for (const revision of revisions) { - if (revision.properties.active) { - azure( - "containerapp", - "revision", - "deactivate", - ...appArguments(installation), - "--revision", - revision.name, - ) - } - } - const deadline = performance.now() + 300_000 - // Inactive revisions may still have terminating replicas. - for (const revision of revisions) { - while ( - azure( - "containerapp", - "replica", - "list", - ...appArguments(installation), - "--revision", - revision.name, - ).length - ) { - if (performance.now() >= deadline) { - throw new Error("Old replicas have not stopped. Hub remains in maintenance.") - } - await timers.setTimeout(5_000) - } - } - } async function setup(installation: Installation): Promise { const execution = azure( "containerapp", @@ -240,15 +217,35 @@ export async function deployHub(args: string[] = process.argv.slice(2)): Promise } async function waitReady(installation: Installation): Promise { const app = azure("containerapp", "show", ...appArguments(installation)) - const revision = app.properties.latestRevisionName - azure( + const revision = app?.properties?.latestRevisionName + if (typeof revision !== "string" || !/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(revision)) { + throw new Error( + "Azure did not return a valid latest Hub revision. Keep the release lock and investigate.", + ) + } + const current = azure( "containerapp", "revision", - "activate", + "show", ...appArguments(installation), "--revision", revision, ) + if (current?.name !== revision || typeof current?.properties?.active !== "boolean") { + throw new Error( + "Azure did not confirm the latest Hub revision's activation state. Keep the release lock and investigate.", + ) + } + if (!current.properties.active) { + azureCommand( + "containerapp", + "revision", + "activate", + ...appArguments(installation), + "--revision", + revision, + ) + } const deadline = performance.now() + 300_000 while (performance.now() < deadline) { const latest = azure("containerapp", "show", ...appArguments(installation)) @@ -275,6 +272,7 @@ export async function deployHub(args: string[] = process.argv.slice(2)): Promise } try { + await mkdir(planDirectory, { recursive: true, mode: 0o700 }) terraform(["init", "-input=false", `-backend-config=${resolve(backend)}`]) if (azure<{ id: string }>("account", "show").id !== subscription) { throw new Error("Azure returned a different subscription.") @@ -325,10 +323,23 @@ export async function deployHub(args: string[] = process.argv.slice(2)): Promise ) { throw new Error("A setup execution is still running; do not start another release.") } - await stopHub(installation) + if (installation.app_created) + await stopHub({ + name: installation.app_name, + resourceGroup: installation.resource_group, + command: (...args) => + args[2] === "deactivate" ? azureCommand(...args) : azure(...args), + }) if (operation === "destroy") { plan(installation.app_created, installation, undefined, true) - azure("storage", "blob", "delete", ...blobArguments(installation, true), "--lease-id", lease) + azureCommand( + "storage", + "blob", + "delete", + ...blobArguments(installation, true), + "--lease-id", + lease, + ) lease = undefined apply() console.log("Terraform-tracked installation resources were destroyed.") @@ -342,7 +353,7 @@ export async function deployHub(args: string[] = process.argv.slice(2)): Promise apply() installation = readInstallation() await waitReady(installation) - azure( + azureCommand( "storage", "blob", "lease", diff --git a/infrastructure/azure/deployment-access.tf b/infrastructure/azure/deployment-access.tf new file mode 100644 index 00000000..8a24434f --- /dev/null +++ b/infrastructure/azure/deployment-access.tf @@ -0,0 +1,16 @@ +variable "grant_deployment_storage_access" { + type = bool + default = false + description = "Wizard opt-in: grant the Azure CLI deployment identity Blob data access on this installation account." +} + +data "azurerm_client_config" "deployment" { + count = var.grant_deployment_storage_access ? 1 : 0 +} + +resource "azurerm_role_assignment" "deployment_storage" { + count = var.grant_deployment_storage_access ? 1 : 0 + scope = azurerm_storage_account.artifacts.id + role_definition_name = "Storage Blob Data Contributor" + principal_id = data.azurerm_client_config.deployment[0].object_id +} diff --git a/infrastructure/azure/environment.tf b/infrastructure/azure/environment.tf index 52cd7c4c..d3bb62d2 100644 --- a/infrastructure/azure/environment.tf +++ b/infrastructure/azure/environment.tf @@ -22,4 +22,9 @@ resource "azurerm_container_app_environment" "hub" { minimum_count = 0 maximum_count = 0 } + + lifecycle { + # Azure names its infrastructure group; resetting that name would replace the environment. + ignore_changes = [infrastructure_resource_group_name] + } } diff --git a/infrastructure/azure/foundation/.terraform.lock.hcl b/infrastructure/azure/foundation/.terraform.lock.hcl new file mode 100644 index 00000000..631af00a --- /dev/null +++ b/infrastructure/azure/foundation/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "4.50.0" + constraints = "4.50.0" + hashes = [ + "h1:Q/JIe7B0F7cziHSDBE7Yr+mFxhtNCDe3JjxuobOGh6w=", + "zh:11899f771a9f03c4e5d222d73ce842e930d26af8aaba88b99a2f0638265331dc", + "zh:2450a4258f40f6ba1f6705a10b2bdb6ff2067ab4c2a6938135391df03c974251", + "zh:2754e761370a18dab5748e020f595e08a7859522889e800204e7a5f65b026590", + "zh:6072e006b1c1b5c1e0a39d373bb6beebf5b4dbb619627c510b6a741521d226c6", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:8ddb7263f15bc16f3b85be8377c9e32ffa4db4cf809586d8010633698ff66105", + "zh:96f973bc54db728defca4817fbd9b1a412c538542e497536d1ce5f6be9946801", + "zh:aa519b7bad1e75930d84ec9ea8462ec384ed26a6ea172281cbd93244ecf8059b", + "zh:adbc238c95915d23d610990770f4b83e576311d4114b9662c4e967d0b0065b1b", + "zh:eaf94d50425ad88cdd42170e9737d8cb1a557ca3737cb905ce34e0218ec621f3", + "zh:f7aa0004e92e67fcee5d8725b878c8de7e6db6f0f766cbe2257d49a994b314f7", + "zh:fbb1195ea2784d67c690ed399b27d59d015c417dec21667f07a6aaf4099cd174", + ] +} diff --git a/infrastructure/azure/foundation/main.tf b/infrastructure/azure/foundation/main.tf new file mode 100644 index 00000000..956cd89f --- /dev/null +++ b/infrastructure/azure/foundation/main.tf @@ -0,0 +1,89 @@ +terraform { + required_version = ">= 1.13.0, < 2.0.0" + backend "azurerm" {} + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "= 4.50.0" + } + } +} + +provider "azurerm" { + subscription_id = var.subscription_id + storage_use_azuread = true + resource_provider_registrations = "none" + features { + resource_group { + prevent_deletion_if_contains_resources = true + } + } +} + +variable "subscription_id" { type = string } +variable "tenant_id" { type = string } +variable "name" { type = string } +variable "location" { type = string } +variable "owner" { type = string } +variable "registry_name" { type = string } +variable "vault_name" { type = string } + +data "azurerm_client_config" "deployment" {} + +locals { + tags = { + application = "replica-hub" + installation = var.name + owner = var.owner + managed_by = "terraform" + purpose = "installation-foundation" + } +} + +resource "azurerm_resource_group" "foundation" { + name = "${var.name}-foundation" + location = var.location + tags = local.tags +} + +resource "azurerm_container_registry" "hub" { + name = var.registry_name + resource_group_name = azurerm_resource_group.foundation.name + location = var.location + sku = "Basic" + admin_enabled = false + tags = local.tags +} + +resource "azurerm_key_vault" "hub" { + name = var.vault_name + location = var.location + resource_group_name = azurerm_resource_group.foundation.name + tenant_id = var.tenant_id + sku_name = "standard" + rbac_authorization_enabled = true + soft_delete_retention_days = 90 + purge_protection_enabled = true + tags = local.tags +} + +resource "azurerm_role_assignment" "secrets" { + scope = azurerm_key_vault.hub.id + role_definition_name = "Key Vault Secrets Officer" + principal_id = data.azurerm_client_config.deployment.object_id +} + +resource "azurerm_role_assignment" "image_push" { + scope = azurerm_container_registry.hub.id + role_definition_name = "AcrPush" + principal_id = data.azurerm_client_config.deployment.object_id +} + +output "installation" { + value = { + vault_id = azurerm_key_vault.hub.id + vault_url = azurerm_key_vault.hub.vault_uri + registry_id = azurerm_container_registry.hub.id + registry_url = azurerm_container_registry.hub.login_server + } +} diff --git a/infrastructure/azure/foundation/tests/foundation.tftest.hcl b/infrastructure/azure/foundation/tests/foundation.tftest.hcl new file mode 100644 index 00000000..a8cd130c --- /dev/null +++ b/infrastructure/azure/foundation/tests/foundation.tftest.hcl @@ -0,0 +1,42 @@ +mock_provider "azurerm" { + mock_data "azurerm_client_config" { + defaults = { + object_id = "00000000-0000-0000-0000-000000000001" + } + } + mock_resource "azurerm_container_registry" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-foundation/providers/Microsoft.ContainerRegistry/registries/testhubregistry" + } + } + mock_resource "azurerm_key_vault" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-foundation/providers/Microsoft.KeyVault/vaults/testhubvault" + } + } +} + +variables { + subscription_id = "00000000-0000-0000-0000-000000000000" + tenant_id = "00000000-0000-0000-0000-000000000000" + name = "test-hub" + location = "eastus2" + owner = "test-team" + registry_name = "testhubregistry" + vault_name = "testhubvault" +} + +run "foundation_protects_secrets_and_registry" { + command = plan + assert { + condition = ( + !azurerm_container_registry.hub.admin_enabled && + azurerm_key_vault.hub.rbac_authorization_enabled && + azurerm_key_vault.hub.purge_protection_enabled && + azurerm_key_vault.hub.soft_delete_retention_days == 90 && + azurerm_role_assignment.image_push.role_definition_name == "AcrPush" && + azurerm_role_assignment.secrets.role_definition_name == "Key Vault Secrets Officer" + ) + error_message = "The foundation requires identity-based registry and vault access with recoverable secrets." + } +} diff --git a/infrastructure/azure/main.tf b/infrastructure/azure/main.tf index 38189c1d..425f1839 100644 --- a/infrastructure/azure/main.tf +++ b/infrastructure/azure/main.tf @@ -8,7 +8,7 @@ locals { } resource "azurerm_resource_group" "hub" { - name = "${var.name}-hub" + name = var.resource_group_name == null ? "${var.name}-hub" : var.resource_group_name location = var.location tags = local.tags } diff --git a/infrastructure/azure/networking.tf b/infrastructure/azure/networking.tf index 3af807e5..8b86403f 100644 --- a/infrastructure/azure/networking.tf +++ b/infrastructure/azure/networking.tf @@ -25,6 +25,7 @@ resource "azurerm_subnet" "database" { resource_group_name = azurerm_resource_group.hub.name virtual_network_name = azurerm_virtual_network.hub.name address_prefixes = ["10.40.2.0/24"] + service_endpoints = ["Microsoft.Storage"] delegation { name = "postgresql" service_delegation { diff --git a/infrastructure/azure/outputs.tf b/infrastructure/azure/outputs.tf index 7c361ec2..ef02b51b 100644 --- a/infrastructure/azure/outputs.tf +++ b/infrastructure/azure/outputs.tf @@ -5,9 +5,12 @@ output "installation" { owner = var.owner subscription_id = var.subscription_id resource_group = azurerm_resource_group.hub.name + resource_group_id = azurerm_resource_group.hub.id + app_id = var.deploy_hub ? azurerm_container_app.hub[0].id : null app_name = "${var.name}-hub" setup_job_name = azurerm_container_app_job.setup.name storage_account = azurerm_storage_account.artifacts.name + storage_account_id = azurerm_storage_account.artifacts.id release_container = azurerm_storage_container.releases.name release_blob = azurerm_storage_blob.release_lock.name environment_id = azurerm_container_app_environment.hub.id diff --git a/infrastructure/azure/provision/.terraform.lock.hcl b/infrastructure/azure/provision/.terraform.lock.hcl new file mode 100644 index 00000000..631af00a --- /dev/null +++ b/infrastructure/azure/provision/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "4.50.0" + constraints = "4.50.0" + hashes = [ + "h1:Q/JIe7B0F7cziHSDBE7Yr+mFxhtNCDe3JjxuobOGh6w=", + "zh:11899f771a9f03c4e5d222d73ce842e930d26af8aaba88b99a2f0638265331dc", + "zh:2450a4258f40f6ba1f6705a10b2bdb6ff2067ab4c2a6938135391df03c974251", + "zh:2754e761370a18dab5748e020f595e08a7859522889e800204e7a5f65b026590", + "zh:6072e006b1c1b5c1e0a39d373bb6beebf5b4dbb619627c510b6a741521d226c6", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:8ddb7263f15bc16f3b85be8377c9e32ffa4db4cf809586d8010633698ff66105", + "zh:96f973bc54db728defca4817fbd9b1a412c538542e497536d1ce5f6be9946801", + "zh:aa519b7bad1e75930d84ec9ea8462ec384ed26a6ea172281cbd93244ecf8059b", + "zh:adbc238c95915d23d610990770f4b83e576311d4114b9662c4e967d0b0065b1b", + "zh:eaf94d50425ad88cdd42170e9737d8cb1a557ca3737cb905ce34e0218ec621f3", + "zh:f7aa0004e92e67fcee5d8725b878c8de7e6db6f0f766cbe2257d49a994b314f7", + "zh:fbb1195ea2784d67c690ed399b27d59d015c417dec21667f07a6aaf4099cd174", + ] +} diff --git a/infrastructure/azure/provision/main.tf b/infrastructure/azure/provision/main.tf new file mode 100644 index 00000000..df8c2705 --- /dev/null +++ b/infrastructure/azure/provision/main.tf @@ -0,0 +1,116 @@ +terraform { + required_version = ">= 1.13.0, < 2.0.0" + backend "azurerm" {} + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "= 4.50.0" + } + } +} + +provider "azurerm" { + subscription_id = var.subscription_id + resource_provider_registrations = "none" + features {} +} + +variable "subscription_id" { type = string } +variable "name" { type = string } +variable "location" { type = string } +variable "resource_group" { type = string } +variable "environment_id" { type = string } +variable "registry_id" { type = string } +variable "image" { + type = string + validation { + condition = can(regex("^[a-z0-9][a-z0-9.:/-]+@sha256:[a-f0-9]{64}$", var.image)) + error_message = "Use an immutable registry/repository@sha256 digest for provisioning." + } +} +variable "vault_id" { type = string } +variable "secret_urls" { + type = map(string) + validation { + condition = ( + toset(keys(var.secret_urls)) == toset(["DATABASE_URL", "HUB_MIGRATION_PASSWORD", "HUB_RUNTIME_PASSWORD"]) && + alltrue([for secret in values(var.secret_urls) : + can(regex("^https://[a-zA-Z0-9-]+\\.vault\\.azure\\.net/secrets/[a-zA-Z0-9-]+/[a-fA-F0-9]{32}$", secret)) + ]) + ) + error_message = "Provide exactly three versioned Key Vault references for the administrator URL and two role passwords, never raw values." + } +} + +resource "azurerm_user_assigned_identity" "provision" { + name = "${var.name}-provision" + location = var.location + resource_group_name = var.resource_group +} + +resource "azurerm_role_assignment" "secrets" { + for_each = var.secret_urls + scope = "${var.vault_id}/secrets/${split("/", each.value)[4]}" + role_definition_name = "Key Vault Secrets User" + principal_id = azurerm_user_assigned_identity.provision.principal_id +} + +resource "azurerm_role_assignment" "image" { + scope = var.registry_id + role_definition_name = "AcrPull" + principal_id = azurerm_user_assigned_identity.provision.principal_id +} + +resource "azurerm_container_app_job" "provision" { + name = "${var.name}-provision" + location = var.location + resource_group_name = var.resource_group + container_app_environment_id = var.environment_id + workload_profile_name = "Consumption" + replica_timeout_in_seconds = 600 + replica_retry_limit = 0 + + manual_trigger_config { + parallelism = 1 + replica_completion_count = 1 + } + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.provision.id] + } + registry { + server = split("/", var.image)[0] + identity = azurerm_user_assigned_identity.provision.id + } + dynamic "secret" { + for_each = var.secret_urls + content { + name = lower(replace(secret.key, "_", "-")) + key_vault_secret_id = secret.value + identity = azurerm_user_assigned_identity.provision.id + } + } + template { + container { + name = "provision" + image = var.image + cpu = 0.5 + memory = "1Gi" + command = ["node", "dist/server/scripts/provision-database-access/index.js"] + env { + name = "NODE_ENV" + value = "production" + } + dynamic "env" { + for_each = var.secret_urls + content { + name = env.key + secret_name = lower(replace(env.key, "_", "-")) + } + } + } + } + depends_on = [azurerm_role_assignment.secrets, azurerm_role_assignment.image] +} + +output "job_name" { value = azurerm_container_app_job.provision.name } diff --git a/infrastructure/azure/provision/tests/provision.tftest.hcl b/infrastructure/azure/provision/tests/provision.tftest.hcl new file mode 100644 index 00000000..cafcc861 --- /dev/null +++ b/infrastructure/azure/provision/tests/provision.tftest.hcl @@ -0,0 +1,43 @@ +mock_provider "azurerm" { + override_during = plan + mock_resource "azurerm_user_assigned_identity" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.ManagedIdentity/userAssignedIdentities/provision" + principal_id = "00000000-0000-0000-0000-000000000001" + } + } +} + +variables { + subscription_id = "00000000-0000-0000-0000-000000000000" + name = "test-hub" + location = "eastus2" + resource_group = "test-hub" + environment_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.App/managedEnvironments/test" + registry_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-foundation/providers/Microsoft.ContainerRegistry/registries/testregistry" + image = "testregistry.azurecr.io/replica-hub@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + vault_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-foundation/providers/Microsoft.KeyVault/vaults/testvault" + secret_urls = { + DATABASE_URL = "https://testvault.vault.azure.net/secrets/postgres-admin-url/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + HUB_MIGRATION_PASSWORD = "https://testvault.vault.azure.net/secrets/postgres-migration-password/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + HUB_RUNTIME_PASSWORD = "https://testvault.vault.azure.net/secrets/postgres-runtime-password/cccccccccccccccccccccccccccccccc" + } +} + +run "fresh_database_job_is_explicit_and_has_separate_access" { + command = plan + assert { + condition = ( + azurerm_container_app_job.provision.replica_retry_limit == 0 && + azurerm_container_app_job.provision.manual_trigger_config[0].parallelism == 1 && + azurerm_container_app_job.provision.template[0].container[0].command == tolist(["node", "dist/server/scripts/provision-database-access/index.js"]) && + length(azurerm_container_app_job.provision.secret) == 3 && + length(azurerm_role_assignment.secrets) == 3 && + alltrue([for secret in azurerm_container_app_job.provision.secret : + secret.identity == azurerm_user_assigned_identity.provision.id && + (secret.value == null || secret.value == "") + ]) + ) + error_message = "Fresh provisioning must use its own identity and Key Vault references, and never retry automatically." + } +} diff --git a/infrastructure/azure/stop-hub/index.test.ts b/infrastructure/azure/stop-hub/index.test.ts new file mode 100644 index 00000000..7f4138ef --- /dev/null +++ b/infrastructure/azure/stop-hub/index.test.ts @@ -0,0 +1,224 @@ +import assert from "node:assert/strict" +import { afterEach, it, mock } from "node:test" +import timers from "node:timers/promises" + +import { stopHub } from "./index.ts" + +afterEach(() => { + mock.restoreAll() +}) + +function option(args: string[], name: string) { + return args[args.indexOf(name) + 1] +} + +for (const adapter of ["synchronous", "asynchronous"]) { + it(`deactivates only active revisions and drains active and inactive replicas with a ${adapter} command adapter`, async () => { + const events: string[][] = [] + const remaining = new Map([ + ["hub-current", 1], + ["hub-old", 2], + ["hub-empty", 0], + ]) + const sleep = mock.method(timers, "setTimeout", async () => undefined) + const execute = (...args: string[]) => { + events.push(args) + if (args.slice(0, 3).join(" ") === "containerapp revision list") + return [ + { name: "hub-current", properties: { active: true } }, + { name: "hub-old", properties: { active: false } }, + { name: "hub-empty", properties: { active: true } }, + ] + if (args[2] === "deactivate") return null + if (args[1] === "replica") { + const revision = option(args, "--revision") + const count = remaining.get(revision)! + remaining.set(revision, count - 1) + return count > 0 ? [{ name: `${revision}-replica` }] : [] + } + throw new Error(`Unexpected command ${args.join(" ")}`) + } + const command = + adapter === "synchronous" ? execute : async (...args: string[]) => execute(...args) + + await stopHub({ name: "test-hub-hub", resourceGroup: "test-group", command }) + + const deactivations = events.filter((args) => args[2] === "deactivate") + assert.deepEqual( + deactivations.map((args) => option(args, "--revision")), + ["hub-current", "hub-empty"], + ) + const replicaReads = events.filter((args) => args[1] === "replica") + assert.deepEqual( + replicaReads.map((args) => option(args, "--revision")), + ["hub-current", "hub-current", "hub-old", "hub-old", "hub-old", "hub-empty"], + ) + assert.ok(events.indexOf(replicaReads[0]!) > events.indexOf(deactivations.at(-1)!)) + for (const args of events) { + assert.equal(option(args, "--name"), "test-hub-hub") + assert.equal(option(args, "--resource-group"), "test-group") + } + assert.deepEqual( + sleep.mock.calls.map(({ arguments: [milliseconds] }) => milliseconds), + [5_000, 5_000, 5_000], + ) + assert.equal( + events.some( + (args) => args.includes("delete") || args.includes("activate") || args.includes("job"), + ), + false, + ) + }) +} + +for (const { description, revisions } of [ + { description: "a missing list", revisions: null }, + { + description: "a missing activity flag", + revisions: [ + { name: "hub-active", properties: { active: true } }, + { name: "hub-unknown", properties: {} }, + ], + }, + { + description: "a string activity flag", + revisions: [ + { name: "hub-active", properties: { active: true } }, + { name: "hub-unknown", properties: { active: "false" } }, + ], + }, + { + description: "a missing revision name", + revisions: [ + { name: "hub-active", properties: { active: true } }, + { properties: { active: false } }, + ], + }, + { + description: "an invalid revision name", + revisions: [ + { name: "hub-active", properties: { active: true } }, + { name: "hub-unknown\nother", properties: { active: false } }, + ], + }, +]) { + it(`validates all revisions before deactivation when Azure returns ${description}`, async () => { + const command = mock.fn(() => revisions) + + await assert.rejects( + stopHub({ name: "hub", resourceGroup: "group", command }), + /revision|metadata/i, + ) + + assert.equal(command.mock.callCount(), 1) + assert.deepEqual(command.mock.calls[0].arguments, [ + "containerapp", + "revision", + "list", + "--name", + "hub", + "--resource-group", + "group", + ]) + }) +} + +it("completes without deactivation or polling for an empty revision inventory", async () => { + const command = mock.fn(() => []) + const sleep = mock.method(timers, "setTimeout", async () => undefined) + + await stopHub({ name: "hub", resourceGroup: "group", command }) + + assert.equal(command.mock.callCount(), 1) + assert.equal(sleep.mock.callCount(), 0) +}) + +for (const source of ["deactivation", "replica inspection", "cancellation"]) { + it(`propagates ${source} failure without automatic activation or other recovery mutations`, async () => { + const failure = + source === "cancellation" + ? Object.assign(new Error("Cancelled"), { cancelled: true }) + : new Error("Azure operation failed") + const events: string[][] = [] + const command = mock.fn((...args: string[]) => { + events.push(args) + if (args[2] === "list" && args[1] === "revision") + return [ + { name: "hub-first", properties: { active: true } }, + { name: "hub-second", properties: { active: true } }, + ] + if ( + (source === "replica inspection" && args[1] === "replica") || + (source !== "replica inspection" && args[2] === "deactivate") + ) + throw failure + return null + }) + + await assert.rejects( + stopHub({ name: "hub", resourceGroup: "group", command }), + (error: unknown) => error === failure, + ) + + assert.equal( + events.some( + (args) => + args.includes("activate") || + args.includes("delete") || + args.includes("job") || + args.includes("lease"), + ), + false, + ) + if (source !== "replica inspection") assert.equal(events.length, 2) + }) +} + +it("does not interpret an unknown replica response as successful shutdown", async () => { + const command = mock.fn((...args: string[]) => + args[1] === "revision" + ? [{ name: "hub-old", properties: { active: false } }] + : { replicas: [] }, + ) + + await assert.rejects( + stopHub({ name: "hub", resourceGroup: "group", command }), + /replica.*list|shutdown.*verif/i, + ) + + assert.equal(command.mock.callCount(), 2) +}) + +it("shares the five-minute drain deadline across active and inactive revisions", async () => { + let now = 0 + mock.method(performance, "now", () => now) + const sleep = mock.method(timers, "setTimeout", async () => { + now += 5_000 + }) + const events: string[][] = [] + const command = mock.fn((...args: string[]) => { + events.push(args) + if (args[1] === "revision" && args[2] === "list") + return [ + { name: "hub-current", properties: { active: true } }, + { name: "hub-old", properties: { active: false } }, + ] + if (args[2] === "deactivate") return null + if (option(args, "--revision") === "hub-current" && now >= 200_000) return [] + return [{ name: "still-running" }] + }) + + await assert.rejects( + stopHub({ name: "hub", resourceGroup: "group", command }), + /replicas.*not stopped|timeout|maintenance/i, + ) + + assert.equal(now, 300_000) + assert.equal(sleep.mock.callCount(), 60) + assert.ok(events.at(-1)?.includes("hub-old")) + assert.equal(events.filter((args) => args[2] === "deactivate").length, 1) + assert.equal( + events.some((args) => args.includes("activate")), + false, + ) +}) diff --git a/infrastructure/azure/stop-hub/index.ts b/infrastructure/azure/stop-hub/index.ts new file mode 100644 index 00000000..ad3ec472 --- /dev/null +++ b/infrastructure/azure/stop-hub/index.ts @@ -0,0 +1,52 @@ +import timers from "node:timers/promises" + +export async function stopHub({ + name, + resourceGroup, + command, +}: { + name: string + resourceGroup: string + command(...args: string[]): unknown | Promise +}): Promise { + const args = ["--name", name, "--resource-group", resourceGroup] + const revisions: unknown = await command("containerapp", "revision", "list", ...args) + const record = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + if ( + !Array.isArray(revisions) || + !revisions.every( + (value): value is { name: string; properties: { active: boolean } } => + record(value) && + typeof value.name === "string" && + /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(value.name) && + record(value.properties) && + typeof value.properties.active === "boolean", + ) + ) + throw new Error("Azure did not return valid Hub revision metadata. Shutdown was not verified.") + + for (const revision of revisions) { + if (revision.properties.active) + await command("containerapp", "revision", "deactivate", ...args, "--revision", revision.name) + } + const deadline = performance.now() + 300_000 + for (const revision of revisions) { + while (true) { + const replicas = await command( + "containerapp", + "replica", + "list", + ...args, + "--revision", + revision.name, + ) + if (!Array.isArray(replicas)) + throw new Error("Azure did not return a Hub replica list. Shutdown was not verified.") + if (performance.now() >= deadline) + throw new Error("Old replicas have not stopped. Hub remains in maintenance.") + if (!replicas.length) break + await timers.setTimeout(5_000) + } + } +} diff --git a/infrastructure/azure/storage.tf b/infrastructure/azure/storage.tf index d56914ee..29e0de14 100644 --- a/infrastructure/azure/storage.tf +++ b/infrastructure/azure/storage.tf @@ -26,12 +26,14 @@ resource "azurerm_storage_container" "artifacts" { name = "artifacts" storage_account_id = azurerm_storage_account.artifacts.id container_access_type = "private" + depends_on = [azurerm_role_assignment.deployment_storage] } resource "azurerm_storage_container" "releases" { name = "releases" storage_account_id = azurerm_storage_account.artifacts.id container_access_type = "private" + depends_on = [azurerm_role_assignment.deployment_storage] } resource "azurerm_storage_blob" "release_lock" { diff --git a/infrastructure/azure/tests/deployment.tftest.hcl b/infrastructure/azure/tests/deployment.tftest.hcl index 36ebac9f..54531f19 100644 --- a/infrastructure/azure/tests/deployment.tftest.hcl +++ b/infrastructure/azure/tests/deployment.tftest.hcl @@ -1,5 +1,15 @@ mock_provider "azurerm" { override_during = plan + mock_resource "azurerm_resource_group" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub" + } + } + mock_resource "azurerm_container_app" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.App/containerApps/test-hub-hub" + } + } mock_resource "azurerm_virtual_network" { defaults = { id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.Network/virtualNetworks/test" @@ -45,6 +55,23 @@ mock_provider "azurerm" { } } +mock_provider "azurerm" { + alias = "rerun" + override_during = apply + + mock_resource "azurerm_container_app_environment" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.App/managedEnvironments/test" + } + } + mock_resource "azurerm_postgresql_flexible_server" { + defaults = { + id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.DBforPostgreSQL/flexibleServers/test" + fqdn = "test.postgres.database.azure.com" + } + } +} + override_resource { target = azurerm_user_assigned_identity.hub override_during = plan @@ -89,6 +116,16 @@ run "bootstrap_keeps_hub_stopped" { condition = length(azurerm_container_app.hub) == 0 error_message = "Bootstrap must not create a combined process before setup." } + assert { + condition = ( + output.installation.app_id == null && + output.installation.resource_group_id == azurerm_resource_group.hub.id && + output.installation.resource_group_id == "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub" && + output.installation.storage_account_id == azurerm_storage_account.artifacts.id && + output.installation.storage_account_id == "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.Storage/storageAccounts/testhub" + ) + error_message = "Bootstrap coordinates must expose authoritative group and storage IDs without claiming a deployed app." + } assert { condition = ( azurerm_container_app_job.setup.template[0].container[0].command == tolist(["node", "dist/server/scripts/setup/index.js"]) && @@ -110,6 +147,10 @@ run "durable_state_requires_private_authenticated_access" { ) error_message = "PostgreSQL must be private, TLS-only, and retain recoverable backups." } + assert { + condition = contains(coalesce(azurerm_subnet.database.service_endpoints, []), "Microsoft.Storage") + error_message = "The database subnet must retain Microsoft.Storage for PostgreSQL WAL archival." + } assert { condition = ( !azurerm_storage_account.artifacts.allow_nested_items_to_be_public && @@ -132,6 +173,13 @@ run "combined_release_uses_one_image_and_bounded_shutdown" { acr_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/existing/providers/Microsoft.ContainerRegistry/registries/existing" } + assert { + condition = ( + output.installation.app_id == azurerm_container_app.hub[0].id && + output.installation.app_id == "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.App/containerApps/test-hub-hub" + ) + error_message = "A deployed app must expose its actual Azure resource ID for the deployment report." + } assert { condition = ( azurerm_container_app.hub[0].template[0].min_replicas == 1 && @@ -261,3 +309,69 @@ run "hostname_without_certificate_is_rejected" { } expect_failures = [var.certificate_id] } + +run "azure_assigned_resource_state" { + command = apply + state_key = "azure-assigned" + + providers = { + azurerm = azurerm.rerun + } + + # Both properties are optional, not computed, so provider mocks cannot populate + # them. Seed only these resources through a mocked apply, never a state edit. + module { + source = "./tests/fixtures/azure-assigned" + } + + assert { + condition = ( + azurerm_container_app_environment.hub.infrastructure_resource_group_name == "ME_test-hub-environment_test-hub_eastus2" && + azurerm_postgresql_flexible_server.database.zone == "2" + ) + error_message = "The mock apply must retain Azure-assigned properties before testing a rerun." + } +} + +run "rerun_retains_azure_assigned_resource_properties" { + command = plan + state_key = "azure-assigned" + + providers = { + azurerm = azurerm.rerun + } + + assert { + condition = azurerm_container_app_environment.hub.infrastructure_resource_group_name == "ME_test-hub-environment_test-hub_eastus2" + error_message = "Rerunning must retain the Azure-owned infrastructure resource group." + } + assert { + condition = azurerm_postgresql_flexible_server.database.zone == "2" + error_message = "Rerunning must retain the Azure-assigned PostgreSQL zone." + } +} + +run "rerun_still_plans_configured_resource_changes" { + command = plan + state_key = "azure-assigned" + + providers = { + azurerm = azurerm.rerun + } + + variables { + owner = "new-owning-team" + postgres_sku = "B_Standard_B2s" + backup_retention_days = 21 + } + + assert { + condition = ( + azurerm_container_app_environment.hub.tags.owner == "new-owning-team" && + azurerm_postgresql_flexible_server.database.tags.owner == "new-owning-team" && + azurerm_postgresql_flexible_server.database.sku_name == "B_Standard_B2s" && + azurerm_postgresql_flexible_server.database.backup_retention_days == 21 + ) + error_message = "Ignoring service-assigned placement must not hide configured resource changes." + } +} diff --git a/infrastructure/azure/tests/fixtures/azure-assigned/main.tf b/infrastructure/azure/tests/fixtures/azure-assigned/main.tf new file mode 100644 index 00000000..a61e3e53 --- /dev/null +++ b/infrastructure/azure/tests/fixtures/azure-assigned/main.tf @@ -0,0 +1,46 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + } + } +} + +resource "azurerm_container_app_environment" "hub" { + name = "test-hub-environment" + resource_group_name = "test-hub-hub" + location = "eastus2" + infrastructure_resource_group_name = "ME_test-hub-environment_test-hub_eastus2" + tags = { + application = "replica-hub" + installation = "test-hub" + owner = "test-team" + managed_by = "terraform" + } + + workload_profile { + name = "Consumption" + workload_profile_type = "Consumption" + minimum_count = 0 + maximum_count = 0 + } +} + +resource "azurerm_postgresql_flexible_server" "database" { + name = "test-hub-postgres" + resource_group_name = "test-hub-hub" + location = "eastus2" + version = "17" + administrator_login = "replicaadmin" + administrator_password = "NotARealPassword-12345" + sku_name = "B_Standard_B1ms" + storage_mb = 32768 + backup_retention_days = 14 + zone = "2" + tags = { + application = "replica-hub" + installation = "test-hub" + owner = "test-team" + managed_by = "terraform" + } +} diff --git a/infrastructure/azure/variables.tf b/infrastructure/azure/variables.tf index 360e13bb..c9f63ec1 100644 --- a/infrastructure/azure/variables.tf +++ b/infrastructure/azure/variables.tf @@ -18,6 +18,12 @@ variable "location" { description = "Azure region supporting Container Apps and PostgreSQL Flexible Server." } +variable "resource_group_name" { + type = string + default = null + description = "Optional dedicated installation resource group name. Existing unmanaged groups are not adopted automatically." +} + variable "owner" { type = string description = "Team responsible for cost, releases, and recovery." diff --git a/infrastructure/azure/versions.tf b/infrastructure/azure/versions.tf index a93abfa8..e0a1f6ff 100644 --- a/infrastructure/azure/versions.tf +++ b/infrastructure/azure/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = "~> 1.13.0" + required_version = ">= 1.13.0, < 2.0.0" backend "azurerm" {} diff --git a/infrastructure/installer/command/describe/index.ts b/infrastructure/installer/command/describe/index.ts new file mode 100644 index 00000000..d4960d11 --- /dev/null +++ b/infrastructure/installer/command/describe/index.ts @@ -0,0 +1,76 @@ +const commands: Record> = { + az: { + version: "Check Azure CLI", + "account list": "List authorized Azure subscriptions", + "account show": "Verify the Azure deployment target", + "account get-access-token": "Read the Azure deployment identity", + rest: "Query Azure resource metadata", + "provider show": "Check Azure resource provider", + "provider register": "Register Azure resource provider", + "group exists": "Check Azure resource group existence", + "group show": "Read Azure resource group ownership", + "group create": "Create Azure resource group", + "storage account list": "List state storage accounts", + "storage account create": "Create protected state storage", + "storage account show": "Inspect state storage", + "storage account update": "Update state storage access", + "storage account blob-service-properties update": "Configure state storage recovery", + "storage container create": "Prepare the Terraform state container", + "storage blob exists": "Check the installation lock blob", + "storage blob show": "Inspect the deployment lease", + "storage blob list": "Inspect Terraform state leases", + "storage blob upload": "Create the installation lock blob", + "storage blob lease acquire": "Acquire the deployment lease", + "storage blob lease release": "Release the deployment lease", + "storage blob lease break": "Recover the selected deployment lease", + "role assignment list": "Check Azure role assignments", + "role assignment create": "Grant Azure resource access", + "keyvault secret list": "Check Key Vault secret access", + "keyvault secret show": "Read an installation secret", + "keyvault secret set": "Store an installation secret", + "acr login": "Sign in to the container registry", + "containerapp job execution list": "Inspect database job executions", + "containerapp job list": "Inspect deployment jobs", + "containerapp list": "Inspect deployed Hub apps", + "containerapp show": "Inspect the deployed Hub app", + "containerapp revision list": "Inspect Hub revision activity", + "containerapp revision show": "Inspect Hub revision activation", + "containerapp revision activate": "Activate Hub revision", + "containerapp revision deactivate": "Deactivate Hub revision", + "containerapp replica list": "Inspect remaining Hub replicas", + "containerapp job execution show": "Check database job progress", + "containerapp job start": "Start the database job", + }, + terraform: { + version: "Check Terraform", + init: "Initialize Terraform", + "state list": "Inspect Terraform ownership", + output: "Read Terraform outputs", + plan: "Plan Terraform changes", + show: "Inspect the Terraform plan", + apply: "Apply Terraform changes", + }, + docker: { + info: "Check Docker", + "buildx version": "Check Docker Buildx", + "buildx build": "Build and publish the Hub image", + }, + git: { + "rev-parse": "Read the source revision", + status: "Check the source working tree", + }, + node: { "": "Run Node.js command" }, +} + +export function describeCommand(executable: string, args: string[]) { + const tool = executable === process.execPath ? "node" : executable + const routes = Object.hasOwn(commands, tool) ? commands[tool] : undefined + const positional = tool === "terraform" && args[0]?.startsWith("-chdir=") ? args.slice(1) : args + for (const [prefix, label] of Object.entries(routes ?? {})) { + const words = prefix ? prefix.split(" ") : [] + if (words.every((word, index) => positional[index] === word)) { + return { name: [tool, ...words].join("."), label } + } + } + return { name: "other", label: "External command" } +} From 4572a86564d67c325b18d9b0a47da7edb9700d1d Mon Sep 17 00:00:00 2001 From: Will Eastler <32074613+weastler@users.noreply.github.com> Date: Fri, 6 Nov 2026 10:20:08 -0800 Subject: [PATCH 2/6] fix(infrastructure): lock Azure providers for Linux CI Record the signed linux_amd64 provider checksum in both optional module locks so readonly initialization remains valid for subsequent validation and mocked tests on CI runners. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- infrastructure/azure/foundation/.terraform.lock.hcl | 1 + infrastructure/azure/provision/.terraform.lock.hcl | 1 + 2 files changed, 2 insertions(+) diff --git a/infrastructure/azure/foundation/.terraform.lock.hcl b/infrastructure/azure/foundation/.terraform.lock.hcl index 631af00a..0bd5c529 100644 --- a/infrastructure/azure/foundation/.terraform.lock.hcl +++ b/infrastructure/azure/foundation/.terraform.lock.hcl @@ -6,6 +6,7 @@ provider "registry.terraform.io/hashicorp/azurerm" { constraints = "4.50.0" hashes = [ "h1:Q/JIe7B0F7cziHSDBE7Yr+mFxhtNCDe3JjxuobOGh6w=", + "h1:j2bM6QYxBw4dKDlNLR2lAWP6Dggv5lhyGU5VREOfaDw=", "zh:11899f771a9f03c4e5d222d73ce842e930d26af8aaba88b99a2f0638265331dc", "zh:2450a4258f40f6ba1f6705a10b2bdb6ff2067ab4c2a6938135391df03c974251", "zh:2754e761370a18dab5748e020f595e08a7859522889e800204e7a5f65b026590", diff --git a/infrastructure/azure/provision/.terraform.lock.hcl b/infrastructure/azure/provision/.terraform.lock.hcl index 631af00a..0bd5c529 100644 --- a/infrastructure/azure/provision/.terraform.lock.hcl +++ b/infrastructure/azure/provision/.terraform.lock.hcl @@ -6,6 +6,7 @@ provider "registry.terraform.io/hashicorp/azurerm" { constraints = "4.50.0" hashes = [ "h1:Q/JIe7B0F7cziHSDBE7Yr+mFxhtNCDe3JjxuobOGh6w=", + "h1:j2bM6QYxBw4dKDlNLR2lAWP6Dggv5lhyGU5VREOfaDw=", "zh:11899f771a9f03c4e5d222d73ce842e930d26af8aaba88b99a2f0638265331dc", "zh:2450a4258f40f6ba1f6705a10b2bdb6ff2067ab4c2a6938135391df03c974251", "zh:2754e761370a18dab5748e020f595e08a7859522889e800204e7a5f65b026590", From 6fe5708b7a0c40968b171991502ac6bca9eee413 Mon Sep 17 00:00:00 2001 From: Will Eastler <32074613+weastler@users.noreply.github.com> Date: Fri, 6 Nov 2026 11:09:56 -0800 Subject: [PATCH 3/6] fix(infrastructure): verify complete Hub shutdown and plan placement Inspect inactive revisions as well as active ones, accept completed drains at the deadline, and resolve sensitive plan paths like Terraform after -chdir. Add independent native regression and safe-command-label coverage to CI. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/hub-infrastructure.yml | 2 +- infrastructure/azure/deploy-hub/index.test.ts | 110 +++++++++++++++++- infrastructure/azure/deploy-hub/index.ts | 2 +- infrastructure/azure/stop-hub/index.test.ts | 47 ++++++-- infrastructure/azure/stop-hub/index.ts | 4 +- .../installer/command/describe/index.test.ts | 99 ++++++++++++++++ 6 files changed, 245 insertions(+), 19 deletions(-) create mode 100644 infrastructure/installer/command/describe/index.test.ts diff --git a/.github/workflows/hub-infrastructure.yml b/.github/workflows/hub-infrastructure.yml index 6884d382..f6448fa7 100644 --- a/.github/workflows/hub-infrastructure.yml +++ b/.github/workflows/hub-infrastructure.yml @@ -42,4 +42,4 @@ jobs: - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 24 - - run: node --test deploy-hub/index.test.ts stop-hub/index.test.ts + - run: node --test deploy-hub/index.test.ts stop-hub/index.test.ts ../installer/command/describe/index.test.ts diff --git a/infrastructure/azure/deploy-hub/index.test.ts b/infrastructure/azure/deploy-hub/index.test.ts index b0a0a70e..853d9951 100644 --- a/infrastructure/azure/deploy-hub/index.test.ts +++ b/infrastructure/azure/deploy-hub/index.test.ts @@ -1,14 +1,41 @@ import assert from "node:assert/strict" import childProcess from "node:child_process" -import { afterEach, describe, it, mock } from "node:test" +import { randomUUID } from "node:crypto" +import { existsSync, mkdirSync, rmSync, statSync, writeFileSync } from "node:fs" +import { basename, dirname, resolve } from "node:path" +import { afterEach, beforeEach, describe, it, mock } from "node:test" import timers from "node:timers/promises" import { inspect } from "node:util" import { describeCommand } from "../../installer/command/describe/index.ts" import { deployHub } from "./index.ts" +const terraformRoot = resolve(import.meta.dirname, "..") +let fixtureDirectory: string +let previousDataDirectory: string | undefined +let previousMask: number +let defaultDirectoryExisted: boolean + +beforeEach(() => { + previousDataDirectory = process.env.TF_DATA_DIR + previousMask = process.umask() + defaultDirectoryExisted = existsSync(resolve(terraformRoot, ".terraform")) + fixtureDirectory = resolve(terraformRoot, `.deploy-hub-test-${randomUUID()}`) + mkdirSync(fixtureDirectory, { mode: 0o700 }) + process.env.TF_DATA_DIR = resolve(fixtureDirectory, "terraform") +}) + afterEach(() => { mock.restoreAll() + if (previousDataDirectory === undefined) delete process.env.TF_DATA_DIR + else process.env.TF_DATA_DIR = previousDataDirectory + process.umask(previousMask) + rmSync(fixtureDirectory, { recursive: true, force: true }) + const callerDirectory = resolve(basename(fixtureDirectory)) + if (callerDirectory !== fixtureDirectory) + rmSync(callerDirectory, { recursive: true, force: true }) + if (!defaultDirectoryExisted) + rmSync(resolve(terraformRoot, ".terraform"), { recursive: true, force: true }) }) function installationScenario({ @@ -26,6 +53,7 @@ function installationScenario({ httpStatuses = [200], commandFailure, commandOutput, + onPlan, }: { appCreated?: boolean setupStatus?: string @@ -36,6 +64,7 @@ function installationScenario({ httpStatuses?: number[] commandFailure?: { args: string[]; error: Error; occurrence?: number } commandOutput?: { args: string[]; stdout: string } + onPlan?: (path: string) => void } = {}) { const installation = { name: "test-hub", @@ -77,6 +106,7 @@ function installationScenario({ return appCreated ? "azurerm_container_app.hub[0]" : "" case "plan": targeted = args.some((argument: string) => argument.startsWith("-target=")) + onPlan?.(args.find((argument) => argument.startsWith("-out="))!.slice(5)) return "" case "show": return JSON.stringify({ @@ -118,11 +148,17 @@ function installationScenario({ } if (args.includes("release") && releaseError) throw releaseError return "" - case "containerapp revision list": - return JSON.stringify([ + case "containerapp revision list": { + const revisions = [ { name: "old-active", properties: { active: true } }, { name: "old-terminating", properties: { active: false } }, - ]) + ] + return JSON.stringify( + args.includes("--all") + ? revisions + : revisions.filter((revision) => revision.properties.active), + ) + } case "containerapp replica list": { const revision = args[args.indexOf("--revision") + 1] const reads = (replicaReads.get(revision) ?? 0) + 1 @@ -170,6 +206,72 @@ function installationScenario({ } describe("deployHub()", () => { + for (const dataDirectory of ["relative", "absolute", "empty", "unset"]) { + it(`keeps sensitive plans in Terraform's data directory when TF_DATA_DIR is ${dataDirectory}`, async () => { + if (dataDirectory === "relative") + process.env.TF_DATA_DIR = `${basename(fixtureDirectory)}/relative` + else if (dataDirectory === "empty") process.env.TF_DATA_DIR = "" + else if (dataDirectory === "unset") delete process.env.TF_DATA_DIR + const expectedDirectory = + dataDirectory === "relative" + ? resolve(fixtureDirectory, "relative") + : dataDirectory === "absolute" + ? resolve(fixtureDirectory, "terraform") + : resolve(terraformRoot, ".terraform") + const directoryExisted = existsSync(expectedDirectory) + const planFiles: { path: string; directoryMode: number; fileMode: number }[] = [] + const { args, events } = installationScenario({ + onPlan(path) { + writeFileSync(path, "synthetic-sensitive-plan") + planFiles.push({ + path, + directoryMode: statSync(dirname(path)).mode & 0o777, + fileMode: statSync(path).mode & 0o777, + }) + }, + }) + process.umask(0o022) + + await deployHub(args) + + assert.equal(process.umask(), 0o022) + assert.equal(planFiles.length, 2) + for (const { path, directoryMode, fileMode } of planFiles) { + assert.equal(dirname(path), expectedDirectory) + if (!directoryExisted) assert.equal(directoryMode, 0o700) + assert.equal(fileMode, 0o600) + assert.equal(existsSync(path), false) + } + assert.ok( + events + .filter((event) => event[0] === "terraform") + .every((event) => event[1] === `-chdir=${terraformRoot}`), + ) + }) + } + + it("deletes a sensitive saved plan and restores the umask after plan inspection fails", async () => { + let planPath: string | undefined + const { args } = installationScenario({ + onPlan(path) { + planPath = path + writeFileSync(path, "synthetic-sensitive-plan") + }, + commandFailure: { + args: [`-chdir=${terraformRoot}`, "show"], + error: new Error("Synthetic plan inspection failure"), + }, + }) + mock.method(console, "error", () => undefined) + process.umask(0o022) + + await assert.rejects(deployHub(args), /Terraform command failed/) + + assert.ok(planPath) + assert.equal(existsSync(planPath), false) + assert.equal(process.umask(), 0o022) + }) + it("drains all old revisions and completes setup before rollout, readiness, and lease release", async () => { const { args, events, replicaReads, commandOptions } = installationScenario() const output: unknown[][] = [] diff --git a/infrastructure/azure/deploy-hub/index.ts b/infrastructure/azure/deploy-hub/index.ts index ce800a4a..b0dfb0cf 100644 --- a/infrastructure/azure/deploy-hub/index.ts +++ b/infrastructure/azure/deploy-hub/index.ts @@ -54,7 +54,7 @@ export async function deployHub(args: string[] = process.argv.slice(2)): Promise const subscriptionId = subscription const variablesPath = resolve(variables) const root = resolve(import.meta.dirname, "..") - const planDirectory = resolve(process.env.TF_DATA_DIR ?? resolve(root, ".terraform")) + const planDirectory = resolve(root, process.env.TF_DATA_DIR || ".terraform") const planPath = resolve(planDirectory, `deployment-${randomUUID()}.tfplan`) const previousMask = process.umask(0o077) let lease: string | undefined diff --git a/infrastructure/azure/stop-hub/index.test.ts b/infrastructure/azure/stop-hub/index.test.ts index 7f4138ef..dbba3d18 100644 --- a/infrastructure/azure/stop-hub/index.test.ts +++ b/infrastructure/azure/stop-hub/index.test.ts @@ -23,12 +23,16 @@ for (const adapter of ["synchronous", "asynchronous"]) { const sleep = mock.method(timers, "setTimeout", async () => undefined) const execute = (...args: string[]) => { events.push(args) - if (args.slice(0, 3).join(" ") === "containerapp revision list") - return [ + if (args.slice(0, 3).join(" ") === "containerapp revision list") { + const revisions = [ { name: "hub-current", properties: { active: true } }, { name: "hub-old", properties: { active: false } }, { name: "hub-empty", properties: { active: true } }, ] + return args.includes("--all") + ? revisions + : revisions.filter((revision) => revision.properties.active) + } if (args[2] === "deactivate") return null if (args[1] === "replica") { const revision = option(args, "--revision") @@ -111,15 +115,10 @@ for (const { description, revisions } of [ ) assert.equal(command.mock.callCount(), 1) - assert.deepEqual(command.mock.calls[0].arguments, [ - "containerapp", - "revision", - "list", - "--name", - "hub", - "--resource-group", - "group", - ]) + assert.deepEqual( + command.mock.calls[0].arguments.filter((arg) => arg !== "--all"), + ["containerapp", "revision", "list", "--name", "hub", "--resource-group", "group"], + ) }) } @@ -222,3 +221,29 @@ it("shares the five-minute drain deadline across active and inactive revisions", false, ) }) + +for (const elapsed of [300_000, 305_000]) { + it(`accepts a drained final revision after ${elapsed} milliseconds of the shared budget`, async () => { + let now = 0 + mock.method(performance, "now", () => now) + const sleep = mock.method(timers, "setTimeout", async () => undefined) + const inspected: string[] = [] + const command = (...args: string[]) => { + if (args[1] === "revision") + return [ + { name: "hub-first", properties: { active: false } }, + { name: "hub-last", properties: { active: false } }, + ] + const revision = option(args, "--revision") + inspected.push(revision) + now += revision === "hub-first" ? 200_000 : elapsed - 200_000 + return [] + } + + await stopHub({ name: "hub", resourceGroup: "group", command }) + + assert.deepEqual(inspected, ["hub-first", "hub-last"]) + assert.equal(now, elapsed) + assert.equal(sleep.mock.callCount(), 0) + }) +} diff --git a/infrastructure/azure/stop-hub/index.ts b/infrastructure/azure/stop-hub/index.ts index ad3ec472..2f42e626 100644 --- a/infrastructure/azure/stop-hub/index.ts +++ b/infrastructure/azure/stop-hub/index.ts @@ -10,7 +10,7 @@ export async function stopHub({ command(...args: string[]): unknown | Promise }): Promise { const args = ["--name", name, "--resource-group", resourceGroup] - const revisions: unknown = await command("containerapp", "revision", "list", ...args) + const revisions: unknown = await command("containerapp", "revision", "list", "--all", ...args) const record = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value) if ( @@ -43,9 +43,9 @@ export async function stopHub({ ) if (!Array.isArray(replicas)) throw new Error("Azure did not return a Hub replica list. Shutdown was not verified.") + if (!replicas.length) break if (performance.now() >= deadline) throw new Error("Old replicas have not stopped. Hub remains in maintenance.") - if (!replicas.length) break await timers.setTimeout(5_000) } } diff --git a/infrastructure/installer/command/describe/index.test.ts b/infrastructure/installer/command/describe/index.test.ts new file mode 100644 index 00000000..11a55cc4 --- /dev/null +++ b/infrastructure/installer/command/describe/index.test.ts @@ -0,0 +1,99 @@ +import assert from "node:assert/strict" +import { it } from "node:test" + +import { describeCommand } from "./index.ts" + +for (const [operation, name, label] of [ + ["containerapp revision list", "az.containerapp.revision.list", "Inspect Hub revision activity"], + [ + "containerapp revision show", + "az.containerapp.revision.show", + "Inspect Hub revision activation", + ], + ["containerapp revision activate", "az.containerapp.revision.activate", "Activate Hub revision"], + [ + "containerapp revision deactivate", + "az.containerapp.revision.deactivate", + "Deactivate Hub revision", + ], + ["containerapp replica list", "az.containerapp.replica.list", "Inspect remaining Hub replicas"], + ["containerapp list", "az.containerapp.list", "Inspect deployed Hub apps"], + ["containerapp show", "az.containerapp.show", "Inspect the deployed Hub app"], + ["storage blob lease acquire", "az.storage.blob.lease.acquire", "Acquire the deployment lease"], + ["storage blob lease release", "az.storage.blob.lease.release", "Release the deployment lease"], + [ + "storage blob lease break", + "az.storage.blob.lease.break", + "Recover the selected deployment lease", + ], + ["containerapp job list", "az.containerapp.job.list", "Inspect deployment jobs"], + [ + "containerapp job execution list", + "az.containerapp.job.execution.list", + "Inspect database job executions", + ], + [ + "containerapp job execution show", + "az.containerapp.job.execution.show", + "Check database job progress", + ], + ["containerapp job start", "az.containerapp.job.start", "Start the database job"], +]) { + it(`describes Azure ${operation} without its resource or secret arguments`, () => { + assert.deepEqual( + describeCommand("az", [ + ...operation.split(" "), + "--name", + "synthetic-private-resource", + "--lease-id", + "synthetic-private-lease", + "--value", + "synthetic-private-secret", + ]), + { name, label }, + ) + }) +} + +for (const [operation, name, label] of [ + ["init", "terraform.init", "Initialize Terraform"], + ["state list", "terraform.state.list", "Inspect Terraform ownership"], + ["output", "terraform.output", "Read Terraform outputs"], + ["plan", "terraform.plan", "Plan Terraform changes"], + ["show", "terraform.show", "Inspect the Terraform plan"], + ["apply", "terraform.apply", "Apply Terraform changes"], +]) { + it(`describes Terraform ${operation} with or without a private -chdir argument`, () => { + for (const prefix of [[], ["-chdir=synthetic-private-directory"]]) { + assert.deepEqual( + describeCommand("terraform", [ + ...prefix, + ...operation.split(" "), + "-var=synthetic-private-secret", + "synthetic-private-plan.tfplan", + ]), + { name, label }, + ) + } + }) +} + +it("describes the current Node executable without exposing the script or its arguments", () => { + assert.deepEqual( + describeCommand(process.execPath, ["synthetic-private-script.ts", "synthetic-private-secret"]), + { name: "node", label: "Run Node.js command" }, + ) +}) + +for (const [executable, args] of [ + ["synthetic-private-executable", ["synthetic-private-secret"]], + ["az", ["containerapp", "revision", "unknown", "synthetic-private-secret"]], + ["terraform", ["-chdir=synthetic-private-directory", "unknown", "synthetic-private-secret"]], +] as const) { + it(`uses a fixed fallback for an unknown ${executable === "synthetic-private-executable" ? "executable" : `${executable} operation`}`, () => { + assert.deepEqual(describeCommand(executable, [...args]), { + name: "other", + label: "External command", + }) + }) +} From c55bf60605a5f4c4de00c88877cb213924105efb Mon Sep 17 00:00:00 2001 From: Will Eastler <32074613+weastler@users.noreply.github.com> Date: Fri, 6 Nov 2026 11:09:56 -0800 Subject: [PATCH 4/6] fix(infrastructure): validate provisioning identity boundaries Reject provisioning secret URLs outside the configured vault and allow public digest images without ACR authentication. Cover secret isolation, optional pull grants, and authenticated deployment storage access using mocked Terraform tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- infrastructure/azure/provision/main.tf | 33 +++- .../provision/tests/provision.tftest.hcl | 169 +++++++++++++++++- .../azure/tests/deployment.tftest.hcl | 34 ++++ 3 files changed, 229 insertions(+), 7 deletions(-) diff --git a/infrastructure/azure/provision/main.tf b/infrastructure/azure/provision/main.tf index df8c2705..88b7d5ee 100644 --- a/infrastructure/azure/provision/main.tf +++ b/infrastructure/azure/provision/main.tf @@ -20,7 +20,11 @@ variable "name" { type = string } variable "location" { type = string } variable "resource_group" { type = string } variable "environment_id" { type = string } -variable "registry_id" { type = string } +variable "registry_id" { + type = string + default = null + description = "Optional existing ACR resource ID. Omit for a public image." +} variable "image" { type = string validation { @@ -28,7 +32,13 @@ variable "image" { error_message = "Use an immutable registry/repository@sha256 digest for provisioning." } } -variable "vault_id" { type = string } +variable "vault_id" { + type = string + validation { + condition = can(regex("^/subscriptions/[^/]+/resourceGroups/[^/]+/providers/Microsoft.KeyVault/vaults/[^/]+$", var.vault_id)) + error_message = "Supply the Key Vault resource ID that owns the provisioning secrets." + } +} variable "secret_urls" { type = map(string) validation { @@ -40,6 +50,15 @@ variable "secret_urls" { ) error_message = "Provide exactly three versioned Key Vault references for the administrator URL and two role passwords, never raw values." } + validation { + condition = ( + !can(regex("^/subscriptions/[^/]+/resourceGroups/[^/]+/providers/Microsoft.KeyVault/vaults/[^/]+$", var.vault_id)) || + alltrue([for secret in values(var.secret_urls) : + lower(try(split("/", secret)[2], "")) == "${lower(try(split("/", var.vault_id)[8], ""))}.vault.azure.net" + ]) + ) + error_message = "Every provisioning secret URL must belong to vault_id." + } } resource "azurerm_user_assigned_identity" "provision" { @@ -56,6 +75,7 @@ resource "azurerm_role_assignment" "secrets" { } resource "azurerm_role_assignment" "image" { + count = var.registry_id == null ? 0 : 1 scope = var.registry_id role_definition_name = "AcrPull" principal_id = azurerm_user_assigned_identity.provision.principal_id @@ -78,9 +98,12 @@ resource "azurerm_container_app_job" "provision" { type = "UserAssigned" identity_ids = [azurerm_user_assigned_identity.provision.id] } - registry { - server = split("/", var.image)[0] - identity = azurerm_user_assigned_identity.provision.id + dynamic "registry" { + for_each = var.registry_id == null ? [] : [var.registry_id] + content { + server = split("/", var.image)[0] + identity = azurerm_user_assigned_identity.provision.id + } } dynamic "secret" { for_each = var.secret_urls diff --git a/infrastructure/azure/provision/tests/provision.tftest.hcl b/infrastructure/azure/provision/tests/provision.tftest.hcl index cafcc861..a11460ff 100644 --- a/infrastructure/azure/provision/tests/provision.tftest.hcl +++ b/infrastructure/azure/provision/tests/provision.tftest.hcl @@ -14,8 +14,7 @@ variables { location = "eastus2" resource_group = "test-hub" environment_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub/providers/Microsoft.App/managedEnvironments/test" - registry_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-foundation/providers/Microsoft.ContainerRegistry/registries/testregistry" - image = "testregistry.azurecr.io/replica-hub@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + image = "ghcr.io/example/hub@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" vault_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-foundation/providers/Microsoft.KeyVault/vaults/testvault" secret_urls = { DATABASE_URL = "https://testvault.vault.azure.net/secrets/postgres-admin-url/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" @@ -26,6 +25,12 @@ variables { run "fresh_database_job_is_explicit_and_has_separate_access" { command = plan + + variables { + registry_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-foundation/providers/Microsoft.ContainerRegistry/registries/testregistry" + image = "testregistry.azurecr.io/replica-hub@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + assert { condition = ( azurerm_container_app_job.provision.replica_retry_limit == 0 && @@ -40,4 +45,164 @@ run "fresh_database_job_is_explicit_and_has_separate_access" { ) error_message = "Fresh provisioning must use its own identity and Key Vault references, and never retry automatically." } + assert { + condition = ( + length(azurerm_container_app_job.provision.registry) == 1 && + one(azurerm_container_app_job.provision.registry).server == "testregistry.azurecr.io" && + one(azurerm_container_app_job.provision.registry).identity == azurerm_user_assigned_identity.provision.id && + length(azurerm_role_assignment.image) == 1 && + alltrue([for grant in azurerm_role_assignment.image : + grant.scope == var.registry_id && + grant.role_definition_name == "AcrPull" && + grant.principal_id == azurerm_user_assigned_identity.provision.principal_id + ]) + ) + error_message = "An explicit ACR needs exactly one registry block and one AcrPull grant for the provisioning identity on that ACR." + } + assert { + condition = ( + azurerm_container_app_job.provision.template[0].container[0].image == var.image && + azurerm_container_app_job.provision.identity[0].identity_ids == toset([azurerm_user_assigned_identity.provision.id]) && + toset([for grant in azurerm_role_assignment.secrets : grant.scope]) == toset([ + "${var.vault_id}/secrets/postgres-admin-url", + "${var.vault_id}/secrets/postgres-migration-password", + "${var.vault_id}/secrets/postgres-runtime-password", + ]) && + alltrue([for grant in azurerm_role_assignment.secrets : + grant.principal_id == azurerm_user_assigned_identity.provision.principal_id && + grant.role_definition_name == "Key Vault Secrets User" + ]) && + toset([for secret in azurerm_container_app_job.provision.secret : secret.key_vault_secret_id]) == toset(values(var.secret_urls)) && + toset([for env in azurerm_container_app_job.provision.template[0].container[0].env : env.name]) == toset([ + "NODE_ENV", "DATABASE_URL", "HUB_MIGRATION_PASSWORD", "HUB_RUNTIME_PASSWORD", + ]) && + alltrue([for name, url in var.secret_urls : + one([for env in azurerm_container_app_job.provision.template[0].container[0].env : env.secret_name if env.name == name]) == + one([for secret in azurerm_container_app_job.provision.secret : secret.name if secret.key_vault_secret_id == url]) + ]) + ) + error_message = "Only the isolated provisioning job may receive its three secret-scoped grants and reference-backed administrator settings." + } +} + +run "public_digest_needs_no_registry_or_pull_grant" { + command = plan + + assert { + condition = ( + var.registry_id == null && + length(azurerm_container_app_job.provision.registry) == 0 && + length(azurerm_role_assignment.image) == 0 && + azurerm_container_app_job.provision.template[0].container[0].image == var.image + ) + error_message = "A public digest must work with registry_id omitted, without registry authentication or an AcrPull grant." + } +} + +run "matching_vault_names_are_case_insensitive" { + command = plan + + variables { + vault_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-foundation/providers/Microsoft.KeyVault/vaults/TESTVAULT" + secret_urls = { + DATABASE_URL = "https://TestVault.vault.azure.net/secrets/postgres-admin-url/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + HUB_MIGRATION_PASSWORD = "https://testvault.vault.azure.net/secrets/postgres-migration-password/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + HUB_RUNTIME_PASSWORD = "https://TESTVAULT.vault.azure.net/secrets/postgres-runtime-password/cccccccccccccccccccccccccccccccc" + } + } + + assert { + condition = ( + length(azurerm_role_assignment.secrets) == 3 && + alltrue([for grant in azurerm_role_assignment.secrets : startswith(grant.scope, "${var.vault_id}/secrets/")]) && + toset([for secret in azurerm_container_app_job.provision.secret : secret.key_vault_secret_id]) == toset(values(var.secret_urls)) + ) + error_message = "Matching DNS vault names must be accepted regardless of case, without changing the configured secret references or grant scopes." + } +} + +run "mutable_image_is_rejected" { + command = plan + + variables { + image = "ghcr.io/example/hub:latest" + } + + expect_failures = [var.image] +} + +run "all_three_provisioning_secrets_are_required" { + command = plan + + variables { + secret_urls = { + HUB_MIGRATION_PASSWORD = "https://testvault.vault.azure.net/secrets/postgres-migration-password/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + HUB_RUNTIME_PASSWORD = "https://testvault.vault.azure.net/secrets/postgres-runtime-password/cccccccccccccccccccccccccccccccc" + } + } + + expect_failures = [var.secret_urls] +} + +run "additional_secret_access_is_rejected" { + command = plan + + variables { + secret_urls = { + DATABASE_URL = "https://testvault.vault.azure.net/secrets/postgres-admin-url/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + HUB_MIGRATION_PASSWORD = "https://testvault.vault.azure.net/secrets/postgres-migration-password/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + HUB_RUNTIME_PASSWORD = "https://testvault.vault.azure.net/secrets/postgres-runtime-password/cccccccccccccccccccccccccccccccc" + EXTRA_SECRET = "https://testvault.vault.azure.net/secrets/extra/dddddddddddddddddddddddddddddddd" + } + } + + expect_failures = [var.secret_urls] +} + +run "unversioned_secret_is_rejected" { + command = plan + + variables { + secret_urls = { + DATABASE_URL = "https://testvault.vault.azure.net/secrets/postgres-admin-url" + HUB_MIGRATION_PASSWORD = "https://testvault.vault.azure.net/secrets/postgres-migration-password/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + HUB_RUNTIME_PASSWORD = "https://testvault.vault.azure.net/secrets/postgres-runtime-password/cccccccccccccccccccccccccccccccc" + } + } + + expect_failures = [var.secret_urls] +} + +run "malformed_vault_resource_id_is_rejected" { + command = plan + + variables { + vault_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-foundation/providers/Microsoft.Storage/storageAccounts/testvault" + } + + expect_failures = [var.vault_id] +} + +run "incomplete_vault_resource_id_is_rejected" { + command = plan + + variables { + vault_id = "testvault" + } + + expect_failures = [var.vault_id] +} + +run "secret_hosts_must_match_the_granted_vault" { + command = plan + + variables { + secret_urls = { + DATABASE_URL = "https://another-vault.vault.azure.net/secrets/postgres-admin-url/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + HUB_MIGRATION_PASSWORD = "https://testvault.vault.azure.net/secrets/postgres-migration-password/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + HUB_RUNTIME_PASSWORD = "https://testvault.vault.azure.net/secrets/postgres-runtime-password/cccccccccccccccccccccccccccccccc" + } + } + + expect_failures = [var.secret_urls] } diff --git a/infrastructure/azure/tests/deployment.tftest.hcl b/infrastructure/azure/tests/deployment.tftest.hcl index 54531f19..eacc8f9f 100644 --- a/infrastructure/azure/tests/deployment.tftest.hcl +++ b/infrastructure/azure/tests/deployment.tftest.hcl @@ -1,5 +1,10 @@ mock_provider "azurerm" { override_during = plan + mock_data "azurerm_client_config" { + defaults = { + object_id = "00000000-0000-0000-0000-000000000005" + } + } mock_resource "azurerm_resource_group" { defaults = { id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/test-hub" @@ -116,6 +121,13 @@ run "bootstrap_keeps_hub_stopped" { condition = length(azurerm_container_app.hub) == 0 error_message = "Bootstrap must not create a combined process before setup." } + assert { + condition = ( + length(data.azurerm_client_config.deployment) == 0 && + length(azurerm_role_assignment.deployment_storage) == 0 + ) + error_message = "Deployment Blob access must remain opt-in without reading or granting a deployment principal by default." + } assert { condition = ( output.installation.app_id == null && @@ -136,6 +148,28 @@ run "bootstrap_keeps_hub_stopped" { } } +run "deployment_blob_access_is_an_explicit_account_scoped_grant" { + command = plan + + variables { + grant_deployment_storage_access = true + } + + assert { + condition = ( + length(data.azurerm_client_config.deployment) == 1 && + length(azurerm_role_assignment.deployment_storage) == 1 && + azurerm_role_assignment.deployment_storage[0].scope == azurerm_storage_account.artifacts.id && + azurerm_role_assignment.deployment_storage[0].role_definition_name == "Storage Blob Data Contributor" && + azurerm_role_assignment.deployment_storage[0].principal_id == data.azurerm_client_config.deployment[0].object_id && + azurerm_role_assignment.deployment_storage[0].principal_id == "00000000-0000-0000-0000-000000000005" && + azurerm_role_assignment.deployment_storage[0].principal_id != azurerm_user_assigned_identity.hub.principal_id && + azurerm_role_assignment.deployment_storage[0].principal_id != azurerm_user_assigned_identity.setup.principal_id + ) + error_message = "Opt-in Blob access must grant only the authenticated deployment principal on the installation storage account, not a workload or the artifacts container." + } +} + run "durable_state_requires_private_authenticated_access" { command = plan From b456ba7f0cebc1537320d0bb8ad565431d6e0ac2 Mon Sep 17 00:00:00 2001 From: Will Eastler <32074613+weastler@users.noreply.github.com> Date: Fri, 6 Nov 2026 11:09:56 -0800 Subject: [PATCH 5/6] docs(infrastructure): clarify deployment prerequisites and proxy choice Document registry and vault provider registration, optional provisioning registry access, and Terraform-relative plan placement. Direct operators to their approved npm proxy rather than a company-specific feed. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- hub/docs/deployment.md | 2 +- infrastructure/README.md | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hub/docs/deployment.md b/hub/docs/deployment.md index bebb70bc..4ef235d6 100644 --- a/hub/docs/deployment.md +++ b/hub/docs/deployment.md @@ -6,7 +6,7 @@ The Hub image runs the frontend, API, and both workers together. It can expose G Build from the repository root with `docker build -t replica-hub:check hub`. When an npm proxy is required, add `--build-arg NPM_REGISTRY=https://YOUR_PROXY/`. This resolves locked dependency versions through that registry without changing the source lockfile; the registry URL must not contain credentials. The image runs as a non-root user and contains compiled application code and migration tooling. It writes no application state to its writable layer. Publish and deploy the same image digest for setup and Hub. -Host npm configuration is not inherited by the image build. Without `NPM_REGISTRY`, the container needs public npm access. Use `--build-arg NPM_REGISTRY=https://packagefeedproxy.microsoft.io/npm/` where public npm access is not allowed. Explicit scoped registry mappings in the copied `.npmrc` still apply. +Host npm configuration is not inherited by the image build. Without `NPM_REGISTRY`, the container needs public npm access. Where that access is not allowed, supply your approved proxy using the build argument above. Explicit scoped registry mappings in the copied `.npmrc` still apply. Publishing is an explicit operator step, not a side effect of CI. From the approved checkout, authenticate to an authorized registry, then run `docker buildx build --platform linux/amd64 --tag YOUR_REGISTRY/replica-hub:YOUR_COMMIT --push hub`. Resolve the published digest with `docker buildx imagetools inspect YOUR_REGISTRY/replica-hub:YOUR_COMMIT` and give deployment the `registry/repository@sha256:...` reference, never the mutable tag. diff --git a/infrastructure/README.md b/infrastructure/README.md index 8ab50651..35a3a187 100644 --- a/infrastructure/README.md +++ b/infrastructure/README.md @@ -18,7 +18,7 @@ Each installation has its own resource group tagged with its application, instal Two optional modules have separate Terraform states and lifecycles. [`azure/foundation/`](azure/foundation/main.tf) creates an installation-owned ACR and RBAC-enabled Key Vault in a separate resource group, with `AcrPush` and `Key Vault Secrets Officer` grants for the deployment identity. The vault has 90-day soft deletion and purge protection. Pass its registry and vault outputs to the main installation; the main module still accepts existing resources and never owns their teardown. -[`azure/provision/`](azure/provision/main.tf) creates a temporary private-network database-role provisioning job and identity in the installation's Container Apps environment. It accepts the same immutable image and exactly three versioned Key Vault references: `DATABASE_URL` for the administrator, `HUB_MIGRATION_PASSWORD`, and `HUB_RUNTIME_PASSWORD`. The job has one replica, no automatic retries, and a ten-minute timeout. An authorized operator must run it once against a fresh database, verify that exact execution succeeded, and remove the temporary job, identity, and grants before release. Do not rerun it against an installed database or grant its administrator secret to Hub or setup. The existing private-network operator script remains an alternative. +[`azure/provision/`](azure/provision/main.tf) creates a temporary private-network database-role provisioning job and identity in the installation's Container Apps environment. It accepts the same immutable image and exactly three versioned Key Vault references: `DATABASE_URL` for the administrator, `HUB_MIGRATION_PASSWORD`, and `HUB_RUNTIME_PASSWORD`. Each secret URL must belong to the supplied `vault_id`. Omit `registry_id` for a public image; an existing ACR requires its resource ID and grants only the provisioning identity pull access. The job has one replica, no automatic retries, and a ten-minute timeout. An authorized operator must run it once against a fresh database, verify that exact execution succeeded, and remove the temporary job, identity, and grants before release. Do not rerun it against an installed database or grant its administrator secret to Hub or setup. The existing private-network operator script remains an alternative. Initialize each optional module against its own protected backend state key, review its declared inputs and Terraform plan, and authorize its apply separately. Never share a state key with the main installation. Neither module starts Hub or replaces the serialized release command. @@ -46,7 +46,7 @@ The deployment identity needs resource creation/update rights for the installati The main module's optional `grant_deployment_storage_access=true` grants the authenticated Terraform deployment identity `Storage Blob Data Contributor` on the installation account. It defaults to `false`; use it only when this account-level grant matches the deployment policy. It does not grant access to the separate state backend. -Register `Microsoft.App`, `Microsoft.OperationalInsights`, `Microsoft.Network`, `Microsoft.Storage`, `Microsoft.DBforPostgreSQL`, and `Microsoft.ManagedIdentity` beforehand. The provider does not register services across the subscription. Allow time for newly assigned RBAC roles to propagate before setup. +Register `Microsoft.App`, `Microsoft.OperationalInsights`, `Microsoft.Network`, `Microsoft.Storage`, `Microsoft.DBforPostgreSQL`, and `Microsoft.ManagedIdentity` beforehand. Before applying the optional foundation module, also register `Microsoft.ContainerRegistry` and `Microsoft.KeyVault`. The providers do not register services across the subscription. Allow time for newly assigned RBAC roles to propagate before setup. Copy [`backend.hcl.example`](azure/backend.hcl.example) to `infrastructure/azure/backend.hcl` and [`installation.tfvars.example`](azure/installation.tfvars.example) to `infrastructure/azure/installation.tfvars`. Both real files are ignored. Replace every placeholder, choose a globally unique name, and resolve an existing image to its real `registry/repository@sha256:...` digest. Use either a public image or an existing ACR; private non-ACR registry credentials and ACR's ABAC repository-permissions mode are not covered by this reference. @@ -94,7 +94,7 @@ The operator must approve the maintenance window and inspect intended configurat Terraform owns the image in both resources. There is deliberately **no** `ignore_changes` or out-of-band `az containerapp update --image`: a later infrastructure apply cannot silently use a stale image. Provisioning and rollout are separated by the two phases above. Never run a full Terraform apply directly on an installed Hub, start a setup job manually, or change revisions in the portal outside this serialized procedure. Terraform state locking alone does not protect setup-to-rollout ordering. The app's `prevent_destroy` guard rejects accidental `deploy_hub=false` on later full plans. -Plans reject resource deletion and replacement during normal releases. A database major upgrade, region move, rename, or replacement needs a reviewed migration/recovery plan, not an override of that guard. The release script stores sensitive plans under `TF_DATA_DIR` when set, otherwise under the module's `.terraform/`, and deletes them on exit. Failures identify a fixed command label and available exit code without printing arguments or tool output. It leaves Hub stopped and retains the infinite lease after any release failure; it never reactivates an older process automatically against a possibly migrated schema. +Plans reject resource deletion and replacement during normal releases. A database major upgrade, region move, rename, or replacement needs a reviewed migration/recovery plan, not an override of that guard. The release script stores sensitive plans under `TF_DATA_DIR` when nonempty, otherwise under the module's `.terraform/`, and deletes them on exit. Relative `TF_DATA_DIR` paths resolve against `infrastructure/azure/`, matching Terraform's `-chdir` behavior; protect that directory if it is outside the module. Failures identify a fixed command label and available exit code without printing arguments or tool output. It leaves Hub stopped and retains the infinite lease after any release failure; it never reactivates an older process automatically against a possibly migrated schema. ### Recover a failed release @@ -156,7 +156,7 @@ terraform init -backend=false -input=false -lockfile=readonly terraform fmt -check -recursive terraform validate terraform test -node --test deploy-hub/index.test.ts stop-hub/index.test.ts +node --test deploy-hub/index.test.ts stop-hub/index.test.ts ../installer/command/describe/index.test.ts ``` Run the same Terraform initialization, formatting, validation, and test commands from `foundation/` and `provision/` to check the optional modules. CI checks each module using both supported Terraform versions in [the workflow matrix](../.github/workflows/hub-infrastructure.yml). From c3a9133a16f65e67be3ea270bf58b4278f1afc34 Mon Sep 17 00:00:00 2001 From: Will Eastler <32074613+weastler@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:08:32 -0800 Subject: [PATCH 6/6] test(hub): await focus after saving the Hub URL Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../components/prototype-hub-connection-dialog/index.test.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hub/client/components/prototype-hub-connection-dialog/index.test.tsx b/hub/client/components/prototype-hub-connection-dialog/index.test.tsx index b87514fe..eada57e0 100644 --- a/hub/client/components/prototype-hub-connection-dialog/index.test.tsx +++ b/hub/client/components/prototype-hub-connection-dialog/index.test.tsx @@ -217,7 +217,9 @@ it("saves an edited URL before connecting and retains invalid drafts and save fa fireEvent.click(screen.getByRole("button", { name: "Save Hub" })) await screen.findByRole("link", { name: "https://new.example.com" }) expect(showModal).toHaveBeenCalledOnce() - expect(screen.getByRole("button", { name: "Edit Hub URL" })).toHaveFocus() + await waitFor(() => { + expect(screen.getByRole("button", { name: "Edit Hub URL" })).toHaveFocus() + }) expect(onSelectHub).toHaveBeenLastCalledWith("https://new.example.com") expect(screen.queryByRole("alert")).not.toBeInTheDocument() fireEvent.click(screen.getByRole("button", { name: "Connect to the Hub" }))