Add GitHub Actions workflows and provisioning scripts for cloud databases

.github/actions/cloud-database/action.yml

@@ -0,0 +1,13 @@
+name: Cloud database provisioner
+description: Provisions a managed database for the run, and deletes it again when the job ends
+inputs:
+  target:
+    description: Which managed database to provision (azure-sqlserver, azure-postgresql, aurora-postgresql or rds-sqlserver)
+    required: true
+  name:
+    description: Name for the resources this run creates, which teardown uses to find them again
+    required: true
+runs:
+  using: node24
+  main: index.mts
+  post: index.mts

.github/actions/cloud-database/aurora-postgresql.mts

@@ -0,0 +1,79 @@
+// Provisions and tears down an Aurora PostgreSQL cluster for one run of the cloud database tests.
+
+import { step, run, capture, newAdminPassword, runnerIpAddress, setPersistenceConnectionString, verifyDatabase, teardownStep } from './common.mts';
+import * as aws from './aws.mts';
+
+const databaseName = 'servicecontrol';
+const adminUser = 'sctestadmin';
+const port = 5432;
+
+function resources(name: string) {
+    return { base: `${name}-aurora`, instance: `${name}-aurora-1` };
+}
+
+async function provision(name: string): Promise<void> {
+    const { base, instance } = resources(name);
+
+    aws.removeStaleSecurityGroups();
+
+    const password = newAdminPassword();
+    const runnerIp = await runnerIpAddress();
+    const { groupId, created } = aws.createSecurityGroup(base);
+    aws.allowRunner(groupId, port, runnerIp);
+
+    const tags = aws.tags(base, created);
+
+    // The engine version is left to the AWS default so that this does not break every time a pinned
+    // minor version is retired.
+    step(`Creating Aurora PostgreSQL cluster ${base}`);
+    run('aws', ['rds', 'create-db-cluster',
+        '--db-cluster-identifier', base,
+        '--engine', 'aurora-postgresql',
+        '--master-username', adminUser,
+        `--master-user-password=${password}`,
+        '--database-name', databaseName,
+        '--vpc-security-group-ids', groupId,
+        '--db-subnet-group-name', aws.dbSubnetGroupName(),
+        '--no-deletion-protection',
+        '--backup-retention-period', '1',
+        ...tags,
+        '--no-cli-pager']);
+
+    step(`Creating instance ${instance}`);
+    run('aws', ['rds', 'create-db-instance',
+        '--db-instance-identifier', instance,
+        '--db-cluster-identifier', base,
+        '--engine', 'aurora-postgresql',
+        '--db-instance-class', 'db.r6g.2xlarge',
+        '--publicly-accessible',
+        ...tags,
+        '--no-cli-pager']);
+
+    step('Waiting for the instance to become available');
+    run('aws', ['rds', 'wait', 'db-instance-available', '--db-instance-identifier', instance]);
+
+    const endpoint = capture('aws', ['rds', 'describe-db-clusters', '--db-cluster-identifier', base, '--query', 'DBClusters[0].Endpoint', '--output', 'text']);
+
+    // Trust Server Certificate because RDS presents an Amazon CA that is not in the runner's trust
+    // store. The connection is still encrypted; only the certificate chain goes unverified.
+    const connectionString = `Host=${endpoint};Port=${port};Database=${databaseName};Username=${adminUser};Password=${password};Ssl Mode=Require;Trust Server Certificate=true;Timeout=60`;
+
+    step('Waiting until the database accepts connections');
+    verifyDatabase('PostgreSql', connectionString);
+
+    setPersistenceConnectionString('PostgreSql', connectionString);
+}
+
+function teardown(name: string): void {
+    const { base, instance } = resources(name);
+
+    teardownStep(`Deleting instance ${instance}`, () =>
+        run('aws', ['rds', 'delete-db-instance', '--db-instance-identifier', instance, '--skip-final-snapshot', '--delete-automated-backups', '--no-cli-pager']));
+
+    teardownStep(`Deleting cluster ${base}`, () =>
+        run('aws', ['rds', 'delete-db-cluster', '--db-cluster-identifier', base, '--skip-final-snapshot', '--no-cli-pager']));
+
+    teardownStep(`Deleting security group ${base}`, () => aws.deleteSecurityGroup(base));
+}
+
+export { provision, teardown };

.github/actions/cloud-database/aws-iam-policy.json

@@ -0,0 +1,59 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "ReadVpcAndSecurityGroups",
+      "Effect": "Allow",
+      "Action": [
+        "ec2:DescribeVpcs",
+        "ec2:DescribeSecurityGroups",
+        "ec2:DescribeSubnets",
+        "ec2:DescribeAvailabilityZones"
+      ],
+      "Resource": "*"
+    },
+    {
+      "Sid": "CreateDefaultVpcOncePerRegion",
+      "Effect": "Allow",
+      "Action": [
+        "ec2:CreateDefaultVpc"
+      ],
+      "Resource": "*"
+    },
+    {
+      "Sid": "ManageTestSecurityGroups",
+      "Effect": "Allow",
+      "Action": [
+        "ec2:CreateSecurityGroup",
+        "ec2:CreateTags",
+        "ec2:AuthorizeSecurityGroupIngress",
+        "ec2:DeleteSecurityGroup"
+      ],
+      "Resource": "*"
+    },
+    {
+      "Sid": "ReadRds",
+      "Effect": "Allow",
+      "Action": [
+        "rds:DescribeDBInstances",
+        "rds:DescribeDBClusters",
+        "rds:DescribeDBSubnetGroups"
+      ],
+      "Resource": "*"
+    },
+    {
+      "Sid": "ManageTestDatabases",
+      "Effect": "Allow",
+      "Action": [
+        "rds:CreateDBInstance",
+        "rds:CreateDBCluster",
+        "rds:CreateDBSubnetGroup",
+        "rds:DeleteDBInstance",
+        "rds:DeleteDBCluster",
+        "rds:DeleteDBInstanceAutomatedBackup",
+        "rds:AddTagsToResource"
+      ],
+      "Resource": "*"
+    }
+  ]
+}

.github/actions/cloud-database/aws.mts

@@ -0,0 +1,141 @@
+// Helpers shared by the two AWS targets.
+
+import { step, run, tryRun, capture, captureJson } from './common.mts';
+
+function findDefaultVpc(): string | null {
+    const vpcId = capture('aws', ['ec2', 'describe-vpcs', '--filters', 'Name=isDefault,Values=true', '--query', 'Vpcs[0].VpcId', '--output', 'text']);
+
+    return vpcId && vpcId !== 'None' ? vpcId : null;
+}
+
+// RDS places a publicly accessible instance in the default VPC's subnet group, and an account has at
+// most one default VPC per region. It is account infrastructure rather than this run's, so it is
+// created when missing and never deleted.
+function defaultVpcId(): string {
+    const existing = findDefaultVpc();
+
+    if (existing) {
+        return existing;
+    }
+
+    step('This region has no default VPC, creating one');
+
+    // Both AWS targets provision at the same time, so the other job may have created it between the
+    // lookup above and this call. The re-read below settles who won, which is why the failure is
+    // held rather than thrown.
+    const failure = tryRun('aws', ['ec2', 'create-default-vpc', '--no-cli-pager']);
+    const created = findDefaultVpc();
+
+    if (!created) {
+        throw new Error(`This region has no default VPC and one could not be created: ${failure}`);
+    }
+
+    return created;
+}
+
+function createdTimestamp(): string {
+    return Math.floor(Date.now() / 1000).toString();
+}
+
+const subnetGroup = 'servicecontrol-cloud-tests';
+
+function dbSubnetGroupName(): string {
+    const existing = capture('aws', ['rds', 'describe-db-subnet-groups', '--query', `length(DBSubnetGroups[?DBSubnetGroupName=='${subnetGroup}'])`, '--output', 'text', '--no-cli-pager']);
+
+    if (existing !== '0') {
+        return subnetGroup;
+    }
+
+    const subnets = capture('aws', ['ec2', 'describe-subnets', '--filters', `Name=vpc-id,Values=${defaultVpcId()}`, 'Name=default-for-az,Values=true', '--query', 'Subnets[].SubnetId', '--output', 'text']).split(/\s+/).filter(Boolean);
+
+    if (subnets.length < 2) {
+        throw new Error(`The default VPC has ${subnets.length} default subnet(s), and RDS needs at least two availability zones.`);
+    }
+
+    step(`Creating DB subnet group ${subnetGroup} across ${subnets.length} subnets`);
+    const failure = tryRun('aws', ['rds', 'create-db-subnet-group',
+        '--db-subnet-group-name', subnetGroup,
+        '--db-subnet-group-description', 'ServiceControl cloud database tests',
+        '--subnet-ids', ...subnets,
+        '--no-cli-pager']);
+
+    // The other AWS target provisions at the same time and may have created it in between.
+    if (failure && capture('aws', ['rds', 'describe-db-subnet-groups', '--query', `length(DBSubnetGroups[?DBSubnetGroupName=='${subnetGroup}'])`, '--output', 'text', '--no-cli-pager']) === '0') {
+        throw new Error(`Could not create the DB subnet group: ${failure}`);
+    }
+
+    return subnetGroup;
+}
+
+function createSecurityGroup(name: string): { groupId: string; created: string } {
+    const vpcId = defaultVpcId();
+    const created = createdTimestamp();
+
+    step(`Creating security group ${name} in ${vpcId}`);
+    const groupId = capture('aws', [
+        'ec2', 'create-security-group',
+        '--group-name', name,
+        '--description', 'ServiceControl cloud database tests',
+        '--vpc-id', vpcId,
+        '--tag-specifications', `ResourceType=security-group,Tags=[{Key=sc-cloud-test,Value=true},{Key=run-id,Value=${name}},{Key=created,Value=${created}}]`,
+        '--query', 'GroupId', '--output', 'text'
+    ]);
+
+    return { groupId, created };
+}
+
+function allowRunner(groupId: string, port: number, runnerIp: string): void {
+    step(`Allowing ${runnerIp} on ${port}`);
+    run('aws', ['ec2', 'authorize-security-group-ingress', '--group-id', groupId, '--protocol', 'tcp', '--port', String(port), '--cidr', `${runnerIp}/32`, '--no-cli-pager']);
+}
+
+function deleteSecurityGroup(name: string): void {
+    const groupId = capture('aws', ['ec2', 'describe-security-groups', '--filters', `Name=group-name,Values=${name}`, '--query', 'SecurityGroups[0].GroupId', '--output', 'text']);
+
+    if (!groupId || groupId === 'None') {
+        return;
+    }
+
+    const failure = tryRun('aws', ['ec2', 'delete-security-group', '--group-id', groupId, '--no-cli-pager']);
+
+    if (!failure) {
+        return;
+    }
+
+    if (failure.includes('DependencyViolation')) {
+        step(`Security group ${name} is still held by a database that is deleting, leaving it for a later run to sweep up`);
+        return;
+    }
+
+    throw new Error(failure);
+}
+
+// A security group cannot be deleted while an instance still holds it, and an RDS instance takes
+// minutes to go, so a run can almost never delete its own. Each run clears out the ones earlier runs
+// left instead, which keeps this self healing without a scheduled workflow.
+function removeStaleSecurityGroups(): void {
+    const cutoff = Math.floor(Date.now() / 1000) - 4 * 60 * 60;
+    const tagged = captureJson('aws', ['ec2', 'describe-security-groups', '--filters', 'Name=tag:sc-cloud-test,Values=true', '--no-cli-pager', '--output', 'json']);
+
+    for (const group of tagged.SecurityGroups) {
+        const created = group.Tags?.find((tag: { Key: string; Value: string }) => tag.Key === 'created')?.Value;
+
+        if (!created || Number(created) >= cutoff) {
+            continue;
+        }
+
+        try {
+            run('aws', ['ec2', 'delete-security-group', '--group-id', group.GroupId, '--no-cli-pager']);
+            step(`Deleted stale security group ${group.GroupName}`);
+        } catch {
+            // Still attached to an instance that has not finished deleting. A later run will get it.
+            step(`Stale security group ${group.GroupName} is still in use, leaving it for a later run`);
+        }
+    }
+}
+
+function tags(name: string, created: string): string[] {
+    return ['--tags', 'Key=sc-cloud-test,Value=true', `Key=run-id,Value=${name}`, `Key=created,Value=${created}`];
+}
+
+export { dbSubnetGroupName, createSecurityGroup, allowRunner, deleteSecurityGroup, removeStaleSecurityGroups, tags };

.github/actions/cloud-database/azure-postgresql.mts

@@ -0,0 +1,50 @@
+import { step, run, newAdminPassword, runnerIpAddress, setPersistenceConnectionString, verifyDatabase, teardownStep } from './common.mts';
+import * as azure from './azure.mts';
+
+const databaseName = 'servicecontrol';
+const adminUser = 'sctestadmin';
+
+async function provision(name: string): Promise<void> {
+    const password = newAdminPassword();
+    const runnerIp = await runnerIpAddress();
+    const location = azure.location();
+    const tags = azure.tags(name);
+
+    step(`Creating PostgreSQL flexible server ${name} in ${azure.resourceGroup} (${location}), allowing ${runnerIp}`);
+    run('az', ['postgres', 'flexible-server', 'create',
+        '--name', name,
+        '--resource-group', azure.resourceGroup,
+        '--location', location,
+        '--admin-user', adminUser,
+        `--admin-password=${password}`,
+        '--tier', 'GeneralPurpose',
+        '--sku-name', 'Standard_D8ds_v5',
+        '--storage-size', '512',
+        '--version', '16',
+        '--geo-redundant-backup', 'Disabled',
+        '--public-access', runnerIp,
+        ...tags,
+        '--yes',
+        '--only-show-errors', '--output', 'none']);
+
+    step(`Creating database ${databaseName}`);
+    run('az', ['postgres', 'flexible-server', 'db', 'create',
+        '--name', databaseName,
+        '--resource-group', azure.resourceGroup,
+        '--server-name', name,
+        '--only-show-errors', '--output', 'none']);
+
+    const connectionString = `Host=${name}.postgres.database.azure.com;Port=5432;Database=${databaseName};Username=${adminUser};Password=${password};Ssl Mode=Require;Timeout=60`;
+
+    step('Waiting until the database accepts connections');
+    verifyDatabase('PostgreSql', connectionString);
+
+    setPersistenceConnectionString('PostgreSql', connectionString);
+}
+
+function teardown(name: string): void {
+    teardownStep(`Deleting PostgreSQL flexible server ${name}`, () =>
+        run('az', ['postgres', 'flexible-server', 'delete', '--name', name, '--resource-group', azure.resourceGroup, '--yes', '--only-show-errors']));
+}
+
+export { provision, teardown };