diff --git a/.github/actions/cloud-database/action.yml b/.github/actions/cloud-database/action.yml new file mode 100644 index 0000000000..57375c4938 --- /dev/null +++ b/.github/actions/cloud-database/action.yml @@ -0,0 +1,13 @@ +name: Cloud database provisioner +description: Provisions a managed database for the run, and deletes it again when the job ends +inputs: + target: + description: Which managed database to provision (azure-sqlserver, azure-postgresql, aurora-postgresql or rds-sqlserver) + required: true + name: + description: Name for the resources this run creates, which teardown uses to find them again + required: true +runs: + using: node24 + main: index.mts + post: index.mts diff --git a/.github/actions/cloud-database/aurora-postgresql.mts b/.github/actions/cloud-database/aurora-postgresql.mts new file mode 100644 index 0000000000..faa243609b --- /dev/null +++ b/.github/actions/cloud-database/aurora-postgresql.mts @@ -0,0 +1,79 @@ +// Provisions and tears down an Aurora PostgreSQL cluster for one run of the cloud database tests. + +import { step, run, capture, newAdminPassword, runnerIpAddress, setPersistenceConnectionString, verifyDatabase, teardownStep } from './common.mts'; +import * as aws from './aws.mts'; + +const databaseName = 'servicecontrol'; +const adminUser = 'sctestadmin'; +const port = 5432; + +function resources(name: string) { + return { base: `${name}-aurora`, instance: `${name}-aurora-1` }; +} + +async function provision(name: string): Promise { + const { base, instance } = resources(name); + + aws.removeStaleSecurityGroups(); + + const password = newAdminPassword(); + const runnerIp = await runnerIpAddress(); + const { groupId, created } = aws.createSecurityGroup(base); + aws.allowRunner(groupId, port, runnerIp); + + const tags = aws.tags(base, created); + + // The engine version is left to the AWS default so that this does not break every time a pinned + // minor version is retired. + step(`Creating Aurora PostgreSQL cluster ${base}`); + run('aws', ['rds', 'create-db-cluster', + '--db-cluster-identifier', base, + '--engine', 'aurora-postgresql', + '--master-username', adminUser, + `--master-user-password=${password}`, + '--database-name', databaseName, + '--vpc-security-group-ids', groupId, + '--db-subnet-group-name', aws.dbSubnetGroupName(), + '--no-deletion-protection', + '--backup-retention-period', '1', + ...tags, + '--no-cli-pager']); + + step(`Creating instance ${instance}`); + run('aws', ['rds', 'create-db-instance', + '--db-instance-identifier', instance, + '--db-cluster-identifier', base, + '--engine', 'aurora-postgresql', + '--db-instance-class', 'db.r6g.2xlarge', + '--publicly-accessible', + ...tags, + '--no-cli-pager']); + + step('Waiting for the instance to become available'); + run('aws', ['rds', 'wait', 'db-instance-available', '--db-instance-identifier', instance]); + + const endpoint = capture('aws', ['rds', 'describe-db-clusters', '--db-cluster-identifier', base, '--query', 'DBClusters[0].Endpoint', '--output', 'text']); + + // Trust Server Certificate because RDS presents an Amazon CA that is not in the runner's trust + // store. The connection is still encrypted; only the certificate chain goes unverified. + const connectionString = `Host=${endpoint};Port=${port};Database=${databaseName};Username=${adminUser};Password=${password};Ssl Mode=Require;Trust Server Certificate=true;Timeout=60`; + + step('Waiting until the database accepts connections'); + verifyDatabase('PostgreSql', connectionString); + + setPersistenceConnectionString('PostgreSql', connectionString); +} + +function teardown(name: string): void { + const { base, instance } = resources(name); + + teardownStep(`Deleting instance ${instance}`, () => + run('aws', ['rds', 'delete-db-instance', '--db-instance-identifier', instance, '--skip-final-snapshot', '--delete-automated-backups', '--no-cli-pager'])); + + teardownStep(`Deleting cluster ${base}`, () => + run('aws', ['rds', 'delete-db-cluster', '--db-cluster-identifier', base, '--skip-final-snapshot', '--no-cli-pager'])); + + teardownStep(`Deleting security group ${base}`, () => aws.deleteSecurityGroup(base)); +} + +export { provision, teardown }; diff --git a/.github/actions/cloud-database/aws-iam-policy.json b/.github/actions/cloud-database/aws-iam-policy.json new file mode 100644 index 0000000000..032603043c --- /dev/null +++ b/.github/actions/cloud-database/aws-iam-policy.json @@ -0,0 +1,59 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ReadVpcAndSecurityGroups", + "Effect": "Allow", + "Action": [ + "ec2:DescribeVpcs", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeAvailabilityZones" + ], + "Resource": "*" + }, + { + "Sid": "CreateDefaultVpcOncePerRegion", + "Effect": "Allow", + "Action": [ + "ec2:CreateDefaultVpc" + ], + "Resource": "*" + }, + { + "Sid": "ManageTestSecurityGroups", + "Effect": "Allow", + "Action": [ + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:DeleteSecurityGroup" + ], + "Resource": "*" + }, + { + "Sid": "ReadRds", + "Effect": "Allow", + "Action": [ + "rds:DescribeDBInstances", + "rds:DescribeDBClusters", + "rds:DescribeDBSubnetGroups" + ], + "Resource": "*" + }, + { + "Sid": "ManageTestDatabases", + "Effect": "Allow", + "Action": [ + "rds:CreateDBInstance", + "rds:CreateDBCluster", + "rds:CreateDBSubnetGroup", + "rds:DeleteDBInstance", + "rds:DeleteDBCluster", + "rds:DeleteDBInstanceAutomatedBackup", + "rds:AddTagsToResource" + ], + "Resource": "*" + } + ] +} diff --git a/.github/actions/cloud-database/aws.mts b/.github/actions/cloud-database/aws.mts new file mode 100644 index 0000000000..8c803932c4 --- /dev/null +++ b/.github/actions/cloud-database/aws.mts @@ -0,0 +1,141 @@ +// Helpers shared by the two AWS targets. + +import { step, run, tryRun, capture, captureJson } from './common.mts'; + +function findDefaultVpc(): string | null { + const vpcId = capture('aws', ['ec2', 'describe-vpcs', '--filters', 'Name=isDefault,Values=true', '--query', 'Vpcs[0].VpcId', '--output', 'text']); + + return vpcId && vpcId !== 'None' ? vpcId : null; +} + +// RDS places a publicly accessible instance in the default VPC's subnet group, and an account has at +// most one default VPC per region. It is account infrastructure rather than this run's, so it is +// created when missing and never deleted. +function defaultVpcId(): string { + const existing = findDefaultVpc(); + + if (existing) { + return existing; + } + + step('This region has no default VPC, creating one'); + + // Both AWS targets provision at the same time, so the other job may have created it between the + // lookup above and this call. The re-read below settles who won, which is why the failure is + // held rather than thrown. + const failure = tryRun('aws', ['ec2', 'create-default-vpc', '--no-cli-pager']); + const created = findDefaultVpc(); + + if (!created) { + throw new Error(`This region has no default VPC and one could not be created: ${failure}`); + } + + return created; +} + +function createdTimestamp(): string { + return Math.floor(Date.now() / 1000).toString(); +} + +const subnetGroup = 'servicecontrol-cloud-tests'; + +function dbSubnetGroupName(): string { + const existing = capture('aws', ['rds', 'describe-db-subnet-groups', '--query', `length(DBSubnetGroups[?DBSubnetGroupName=='${subnetGroup}'])`, '--output', 'text', '--no-cli-pager']); + + if (existing !== '0') { + return subnetGroup; + } + + const subnets = capture('aws', ['ec2', 'describe-subnets', '--filters', `Name=vpc-id,Values=${defaultVpcId()}`, 'Name=default-for-az,Values=true', '--query', 'Subnets[].SubnetId', '--output', 'text']).split(/\s+/).filter(Boolean); + + if (subnets.length < 2) { + throw new Error(`The default VPC has ${subnets.length} default subnet(s), and RDS needs at least two availability zones.`); + } + + step(`Creating DB subnet group ${subnetGroup} across ${subnets.length} subnets`); + const failure = tryRun('aws', ['rds', 'create-db-subnet-group', + '--db-subnet-group-name', subnetGroup, + '--db-subnet-group-description', 'ServiceControl cloud database tests', + '--subnet-ids', ...subnets, + '--no-cli-pager']); + + // The other AWS target provisions at the same time and may have created it in between. + if (failure && capture('aws', ['rds', 'describe-db-subnet-groups', '--query', `length(DBSubnetGroups[?DBSubnetGroupName=='${subnetGroup}'])`, '--output', 'text', '--no-cli-pager']) === '0') { + throw new Error(`Could not create the DB subnet group: ${failure}`); + } + + return subnetGroup; +} + +function createSecurityGroup(name: string): { groupId: string; created: string } { + const vpcId = defaultVpcId(); + const created = createdTimestamp(); + + step(`Creating security group ${name} in ${vpcId}`); + const groupId = capture('aws', [ + 'ec2', 'create-security-group', + '--group-name', name, + '--description', 'ServiceControl cloud database tests', + '--vpc-id', vpcId, + '--tag-specifications', `ResourceType=security-group,Tags=[{Key=sc-cloud-test,Value=true},{Key=run-id,Value=${name}},{Key=created,Value=${created}}]`, + '--query', 'GroupId', '--output', 'text' + ]); + + return { groupId, created }; +} + +function allowRunner(groupId: string, port: number, runnerIp: string): void { + step(`Allowing ${runnerIp} on ${port}`); + run('aws', ['ec2', 'authorize-security-group-ingress', '--group-id', groupId, '--protocol', 'tcp', '--port', String(port), '--cidr', `${runnerIp}/32`, '--no-cli-pager']); +} + +function deleteSecurityGroup(name: string): void { + const groupId = capture('aws', ['ec2', 'describe-security-groups', '--filters', `Name=group-name,Values=${name}`, '--query', 'SecurityGroups[0].GroupId', '--output', 'text']); + + if (!groupId || groupId === 'None') { + return; + } + + const failure = tryRun('aws', ['ec2', 'delete-security-group', '--group-id', groupId, '--no-cli-pager']); + + if (!failure) { + return; + } + + if (failure.includes('DependencyViolation')) { + step(`Security group ${name} is still held by a database that is deleting, leaving it for a later run to sweep up`); + return; + } + + throw new Error(failure); +} + +// A security group cannot be deleted while an instance still holds it, and an RDS instance takes +// minutes to go, so a run can almost never delete its own. Each run clears out the ones earlier runs +// left instead, which keeps this self healing without a scheduled workflow. +function removeStaleSecurityGroups(): void { + const cutoff = Math.floor(Date.now() / 1000) - 4 * 60 * 60; + const tagged = captureJson('aws', ['ec2', 'describe-security-groups', '--filters', 'Name=tag:sc-cloud-test,Values=true', '--no-cli-pager', '--output', 'json']); + + for (const group of tagged.SecurityGroups) { + const created = group.Tags?.find((tag: { Key: string; Value: string }) => tag.Key === 'created')?.Value; + + if (!created || Number(created) >= cutoff) { + continue; + } + + try { + run('aws', ['ec2', 'delete-security-group', '--group-id', group.GroupId, '--no-cli-pager']); + step(`Deleted stale security group ${group.GroupName}`); + } catch { + // Still attached to an instance that has not finished deleting. A later run will get it. + step(`Stale security group ${group.GroupName} is still in use, leaving it for a later run`); + } + } +} + +function tags(name: string, created: string): string[] { + return ['--tags', 'Key=sc-cloud-test,Value=true', `Key=run-id,Value=${name}`, `Key=created,Value=${created}`]; +} + +export { dbSubnetGroupName, createSecurityGroup, allowRunner, deleteSecurityGroup, removeStaleSecurityGroups, tags }; diff --git a/.github/actions/cloud-database/azure-postgresql.mts b/.github/actions/cloud-database/azure-postgresql.mts new file mode 100644 index 0000000000..ff1cc7efde --- /dev/null +++ b/.github/actions/cloud-database/azure-postgresql.mts @@ -0,0 +1,50 @@ +import { step, run, newAdminPassword, runnerIpAddress, setPersistenceConnectionString, verifyDatabase, teardownStep } from './common.mts'; +import * as azure from './azure.mts'; + +const databaseName = 'servicecontrol'; +const adminUser = 'sctestadmin'; + +async function provision(name: string): Promise { + const password = newAdminPassword(); + const runnerIp = await runnerIpAddress(); + const location = azure.location(); + const tags = azure.tags(name); + + step(`Creating PostgreSQL flexible server ${name} in ${azure.resourceGroup} (${location}), allowing ${runnerIp}`); + run('az', ['postgres', 'flexible-server', 'create', + '--name', name, + '--resource-group', azure.resourceGroup, + '--location', location, + '--admin-user', adminUser, + `--admin-password=${password}`, + '--tier', 'GeneralPurpose', + '--sku-name', 'Standard_D8ds_v5', + '--storage-size', '512', + '--version', '16', + '--geo-redundant-backup', 'Disabled', + '--public-access', runnerIp, + ...tags, + '--yes', + '--only-show-errors', '--output', 'none']); + + step(`Creating database ${databaseName}`); + run('az', ['postgres', 'flexible-server', 'db', 'create', + '--name', databaseName, + '--resource-group', azure.resourceGroup, + '--server-name', name, + '--only-show-errors', '--output', 'none']); + + const connectionString = `Host=${name}.postgres.database.azure.com;Port=5432;Database=${databaseName};Username=${adminUser};Password=${password};Ssl Mode=Require;Timeout=60`; + + step('Waiting until the database accepts connections'); + verifyDatabase('PostgreSql', connectionString); + + setPersistenceConnectionString('PostgreSql', connectionString); +} + +function teardown(name: string): void { + teardownStep(`Deleting PostgreSQL flexible server ${name}`, () => + run('az', ['postgres', 'flexible-server', 'delete', '--name', name, '--resource-group', azure.resourceGroup, '--yes', '--only-show-errors'])); +} + +export { provision, teardown }; diff --git a/.github/actions/cloud-database/azure-sqlserver.mts b/.github/actions/cloud-database/azure-sqlserver.mts new file mode 100644 index 0000000000..09a4e39a37 --- /dev/null +++ b/.github/actions/cloud-database/azure-sqlserver.mts @@ -0,0 +1,55 @@ +import { step, run, newAdminPassword, runnerIpAddress, setPersistenceConnectionString, verifyDatabase, teardownStep } from './common.mts'; +import * as azure from './azure.mts'; + +const databaseName = 'servicecontrol'; +const adminUser = 'sctestadmin'; + +async function provision(name: string): Promise { + const password = newAdminPassword(); + const runnerIp = await runnerIpAddress(); + const location = azure.location(); + const tags = azure.tags(name); + + step(`Creating SQL server ${name} in ${azure.resourceGroup} (${location})`); + run('az', ['sql', 'server', 'create', + '--name', name, + '--resource-group', azure.resourceGroup, + '--location', location, + '--admin-user', adminUser, + `--admin-password=${password}`, + ...tags, + '--only-show-errors', '--output', 'none']); + + step(`Allowing ${runnerIp} through the server firewall`); + run('az', ['sql', 'server', 'firewall-rule', 'create', + '--name', 'github-runner', + '--resource-group', azure.resourceGroup, + '--server', name, + '--start-ip-address', runnerIp, + '--end-ip-address', runnerIp, + '--only-show-errors', '--output', 'none']); + + step(`Creating database ${databaseName}`); + run('az', ['sql', 'db', 'create', + '--name', databaseName, + '--resource-group', azure.resourceGroup, + '--server', name, + '--service-objective', 'BC_Gen5_8', + '--backup-storage-redundancy', 'Local', + ...tags, + '--only-show-errors', '--output', 'none']); + + const connectionString = `Server=tcp:${name}.database.windows.net,1433;Initial Catalog=${databaseName};User ID=${adminUser};Password=${password};Encrypt=True;TrustServerCertificate=False;Connect Timeout=60`; + + step('Waiting until the database accepts connections'); + verifyDatabase('SqlServer', connectionString); + + setPersistenceConnectionString('SqlServer', connectionString); +} + +function teardown(name: string): void { + teardownStep(`Deleting SQL server ${name}`, () => + run('az', ['sql', 'server', 'delete', '--name', name, '--resource-group', azure.resourceGroup, '--yes', '--only-show-errors'])); +} + +export { provision, teardown }; diff --git a/.github/actions/cloud-database/azure.mts b/.github/actions/cloud-database/azure.mts new file mode 100644 index 0000000000..9d6c76178d --- /dev/null +++ b/.github/actions/cloud-database/azure.mts @@ -0,0 +1,23 @@ +// Helpers shared by the two Azure targets. + +import { capture } from './common.mts'; + +const resourceGroup = 'GitHubActions-RG'; + +function tags(runId: string): string[] { + const created = new Date().toISOString().slice(0, 10); + + return ['--tags', `Created=${created}`, 'Package=ServiceControl', 'RunnerOS=Linux', `RunId=${runId}`]; +} + +function location(): string { + const value = capture('az', ['group', 'show', '--name', resourceGroup, '--query', 'location', '--output', 'tsv']); + + if (!value) { + throw new Error(`Could not read the location of resource group ${resourceGroup}.`); + } + + return value; +} + +export { resourceGroup, tags, location }; diff --git a/.github/actions/cloud-database/common.mts b/.github/actions/cloud-database/common.mts new file mode 100644 index 0000000000..72d5fb9980 --- /dev/null +++ b/.github/actions/cloud-database/common.mts @@ -0,0 +1,115 @@ +// Shared helpers for the target modules beside this file. + +import { execFileSync } from 'node:child_process'; +import { appendFileSync } from 'node:fs'; +import { randomInt } from 'node:crypto'; + +export type Provider = 'SqlServer' | 'PostgreSql'; +export type Target = 'azure-sqlserver' | 'azure-postgresql' | 'aurora-postgresql' | 'rds-sqlserver'; + +function step(message: string): void { + console.log(`==> ${message}`); +} + +// Both CLIs report failure by exiting non-zero, and execFileSync turns that into a throw, so a +// provision that fails half way does not carry on building on top of the failure. +function run(command: string, args: string[]): void { + execFileSync(command, args, { stdio: 'inherit' }); +} + +// Runs a command that is allowed to fail, returning the reason it gave when it does. Reporting the +// CLI's own message beats guessing at the cause in an error string. +function tryRun(command: string, args: string[]): string | null { + try { + execFileSync(command, args, { stdio: ['ignore', 'inherit', 'pipe'], encoding: 'utf8' }); + return null; + } catch (error) { + const stderr = String((error as { stderr?: unknown }).stderr ?? '').trim(); + return stderr.split('\n').filter(Boolean).pop() ?? (error as Error).message; + } +} + +function capture(command: string, args: string[]): string { + return execFileSync(command, args, { encoding: 'utf8' }).trim(); +} + +function captureJson(command: string, args: string[]): any { + return JSON.parse(capture(command, args)); +} + +function newAdminPassword(): string { + const upper = 'ABCDEFGHJKLMNPQRSTUVWXYZ'; + const lower = 'abcdefghijkmnpqrstuvwxyz'; + const digit = '23456789'; + const symbol = '!#$%*()_+'; + const all = upper + lower + digit + symbol; + + const pick = (set: string) => set[randomInt(set.length)]; + const characters = [pick(upper), pick(lower), pick(digit), pick(symbol)]; + + while (characters.length < 28) { + characters.push(pick(all)); + } + + for (let i = characters.length - 1; i > 0; i--) { + const j = randomInt(i + 1); + [characters[i], characters[j]] = [characters[j], characters[i]]; + } + + const letter = characters.findIndex(c => /[A-Za-z]/.test(c)); + [characters[0], characters[letter]] = [characters[letter], characters[0]]; + + const password = characters.join(''); + + console.log(`::add-mask::${password}`); + + return password; +} + +async function runnerIpAddress(): Promise { + const response = await fetch('https://api.ipify.org', { signal: AbortSignal.timeout(30000) }); + + if (!response.ok) { + throw new Error(`Could not determine the runner's public IP address: ${response.status} ${response.statusText}`); + } + + return (await response.text()).trim(); +} + +function setPersistenceConnectionString(provider: Provider, connectionString: string): void { + const name = `ServiceControl_Persistence_${provider}_ConnectionString`; + appendFileSync(process.env.GITHUB_ENV, `${name}=${connectionString}\n`); + step(`Exported ${name}`); +} + +// Waits until the database is really reachable. +function verifyDatabase(provider: Provider, connectionString: string): void { + const script = provider === 'SqlServer' ? './verify-sqlserver.cs' : './verify-postgresql.cs'; + + execFileSync('dotnet', ['run', script, '--', connectionString], { cwd: import.meta.dirname, stdio: 'inherit' }); +} + +// Teardown runs even when provisioning failed part way, so it has to tolerate resources that were +// never created. +function teardownStep(description: string, action: () => void): void { + step(description); + + try { + action(); + } catch (error) { + console.log(`::warning::${description} failed: ${(error as Error).message}`); + } +} + +export { + step, + run, + tryRun, + capture, + captureJson, + newAdminPassword, + runnerIpAddress, + setPersistenceConnectionString, + verifyDatabase, + teardownStep +}; diff --git a/.github/actions/cloud-database/index.mts b/.github/actions/cloud-database/index.mts new file mode 100644 index 0000000000..87c2f8d71e --- /dev/null +++ b/.github/actions/cloud-database/index.mts @@ -0,0 +1,38 @@ +import { appendFileSync } from 'node:fs'; +import type { Target } from './common.mts'; + +const targets: Target[] = ['azure-sqlserver', 'azure-postgresql', 'aurora-postgresql', 'rds-sqlserver']; + +function fail(message: string): never { + console.log(`::error::${message}`); + process.exit(1); +} + +function input(name: string): string { + return process.env[`INPUT_${name.toUpperCase()}`] || fail(`The ${name} input is required.`); +} + +async function main() { + const target = input('target'); + const name = input('name'); + + if (!targets.includes(target as Target)) { + fail(`'${target}' is not a cloud database target. Expected one of ${targets.join(', ')}.`); + } + + const database = await import(`./${target}.mts`); + + // GitHub sets STATE_ variables in the post step from whatever the main step wrote to GITHUB_STATE. + if (process.env.STATE_provisioning) { + database.teardown(name); + return; + } + + appendFileSync(process.env.GITHUB_STATE, `provisioning=${target}\n`); + await database.provision(name); +} + +main().catch(error => { + console.log(`::error::${(error as Error).message}`); + process.exit(1); +}); diff --git a/.github/actions/cloud-database/rds-sqlserver.mts b/.github/actions/cloud-database/rds-sqlserver.mts new file mode 100644 index 0000000000..c9a7672b68 --- /dev/null +++ b/.github/actions/cloud-database/rds-sqlserver.mts @@ -0,0 +1,66 @@ +import { step, run, capture, newAdminPassword, runnerIpAddress, setPersistenceConnectionString, verifyDatabase, teardownStep } from './common.mts'; +import * as aws from './aws.mts'; + +const databaseName = 'servicecontrol'; +const adminUser = 'sctestadmin'; +const port = 1433; + +const engine = 'sqlserver-web'; + +function instanceName(name: string): string { + return `${name}-mssql`; +} + +async function provision(name: string): Promise { + const instance = instanceName(name); + + aws.removeStaleSecurityGroups(); + + const password = newAdminPassword(); + const runnerIp = await runnerIpAddress(); + const { groupId, created } = aws.createSecurityGroup(instance); + aws.allowRunner(groupId, port, runnerIp); + + step(`Creating RDS SQL Server instance ${instance}`); + run('aws', ['rds', 'create-db-instance', + '--db-instance-identifier', instance, + '--engine', engine, + '--db-instance-class', 'db.m5.2xlarge', + '--allocated-storage', '100', + '--storage-type', 'gp3', + '--master-username', adminUser, + `--master-user-password=${password}`, + '--vpc-security-group-ids', groupId, + '--db-subnet-group-name', aws.dbSubnetGroupName(), + '--license-model', 'license-included', + '--publicly-accessible', + '--no-multi-az', + '--backup-retention-period', '0', + ...aws.tags(instance, created), + '--no-cli-pager']); + + step('Waiting for the instance to become available'); + run('aws', ['rds', 'wait', 'db-instance-available', '--db-instance-identifier', instance]); + + const endpoint = capture('aws', ['rds', 'describe-db-instances', '--db-instance-identifier', instance, '--query', 'DBInstances[0].Endpoint.Address', '--output', 'text']); + + // Trust Server Certificate because RDS presents an Amazon CA that is not in the runner's trust + // store. The connection is still encrypted; only the certificate chain goes unverified. + const connectionString = `Server=tcp:${endpoint},${port};Initial Catalog=${databaseName};User ID=${adminUser};Password=${password};Encrypt=True;TrustServerCertificate=True;Connect Timeout=60`; + + step(`Creating database ${databaseName} and verifying Full-Text Search`); + verifyDatabase('SqlServer', connectionString); + + setPersistenceConnectionString('SqlServer', connectionString); +} + +function teardown(name: string): void { + const instance = instanceName(name); + + teardownStep(`Deleting instance ${instance}`, () => + run('aws', ['rds', 'delete-db-instance', '--db-instance-identifier', instance, '--skip-final-snapshot', '--delete-automated-backups', '--no-cli-pager'])); + + teardownStep(`Deleting security group ${instance}`, () => aws.deleteSecurityGroup(instance)); +} + +export { provision, teardown }; diff --git a/.github/actions/cloud-database/verify-postgresql.cs b/.github/actions/cloud-database/verify-postgresql.cs new file mode 100644 index 0000000000..72d55a4f28 --- /dev/null +++ b/.github/actions/cloud-database/verify-postgresql.cs @@ -0,0 +1,43 @@ +#:package Npgsql@10.0.3 + +// Waits until the server actually accepts connections on the test database. + +using Npgsql; + +if (args.Length != 1) +{ + Console.Error.WriteLine("usage: dotnet run verify-postgresql.cs -- "); + return 1; +} + +var builder = new NpgsqlConnectionStringBuilder(args[0]); +var deadline = DateTime.UtcNow.AddMinutes(10); +var attempt = 0; + +while (true) +{ + attempt++; + + try + { + await using var connection = new NpgsqlConnection(builder.ConnectionString); + await connection.OpenAsync(); + + await using var command = connection.CreateCommand(); + command.CommandText = "SELECT current_database()"; + var database = (string)(await command.ExecuteScalarAsync())!; + + Console.WriteLine($"Database '{database}' is reachable on {builder.Host}, after {attempt} attempt(s)."); + return 0; + } + catch (Exception e) when (e is NpgsqlException or TimeoutException && DateTime.UtcNow < deadline) + { + Console.WriteLine($"{builder.Host} is not ready yet (attempt {attempt}): {e.Message.Split('\n')[0]}"); + await Task.Delay(TimeSpan.FromSeconds(10)); + } + catch (Exception e) when (e is NpgsqlException or TimeoutException) + { + Console.Error.WriteLine($"Gave up waiting for {builder.Host} after {attempt} attempts: {e.Message}"); + return 1; + } +} diff --git a/.github/actions/cloud-database/verify-sqlserver.cs b/.github/actions/cloud-database/verify-sqlserver.cs new file mode 100644 index 0000000000..28957078a3 --- /dev/null +++ b/.github/actions/cloud-database/verify-sqlserver.cs @@ -0,0 +1,78 @@ +#:package Microsoft.Data.SqlClient@6.1.1 + +// Waits until the server accepts connections, creates the test database if the provisioning CLI +// could not, and fails the run if the server has no Full-Text Search. + +using Microsoft.Data.SqlClient; + +if (args.Length != 1) +{ + Console.Error.WriteLine("usage: dotnet run verify-sqlserver.cs -- "); + return 1; +} + +var builder = new SqlConnectionStringBuilder(args[0]); +var database = builder.InitialCatalog; + +builder.InitialCatalog = "master"; + +var deadline = DateTime.UtcNow.AddMinutes(10); +var attempt = 0; +SqlConnection master; + +while (true) +{ + attempt++; + + try + { + master = new SqlConnection(builder.ConnectionString); + await master.OpenAsync(); + break; + } + catch (SqlException e) when (DateTime.UtcNow < deadline) + { + Console.WriteLine($"{builder.DataSource} is not reachable yet (attempt {attempt}): {e.Message.Split('\n')[0]}"); + await Task.Delay(TimeSpan.FromSeconds(10)); + } + catch (SqlException e) + { + Console.Error.WriteLine($"Gave up waiting for {builder.DataSource} after {attempt} attempts: {e.Message}"); + return 1; + } +} + +await using (master) +{ + // sys.databases rather than DB_ID: on Azure SQL the master database is a logical one, and + // DB_ID returns null for a database that is sitting right there on the same server. + bool exists; + await using (var lookup = master.CreateCommand()) + { + lookup.CommandText = "SELECT COUNT(*) FROM sys.databases WHERE name = @database"; + lookup.Parameters.AddWithValue("@database", database); + exists = (int)(await lookup.ExecuteScalarAsync())! > 0; + } + + if (!exists) + { + Console.WriteLine($"Creating database '{database}'."); + await using var create = master.CreateCommand(); + create.CommandText = $"CREATE DATABASE [{database}]"; + create.CommandTimeout = 300; + await create.ExecuteNonQueryAsync(); + } + + await using (var fullText = master.CreateCommand()) + { + fullText.CommandText = "SELECT CONVERT(int, ISNULL(SERVERPROPERTY('IsFullTextInstalled'), 0))"; + if ((int)(await fullText.ExecuteScalarAsync())! != 1) + { + Console.Error.WriteLine($"{builder.DataSource} does not have SQL Server Full-Text Search installed, which ServiceControl requires. On RDS, check that the edition supports it and that the option group includes it."); + return 1; + } + } +} + +Console.WriteLine($"Database '{database}' is present on {builder.DataSource} and Full-Text Search is installed, after {attempt} connection attempt(s)."); +return 0; diff --git a/.github/workflows/cloud-database-tests.yml b/.github/workflows/cloud-database-tests.yml new file mode 100644 index 0000000000..11b6344555 --- /dev/null +++ b/.github/workflows/cloud-database-tests.yml @@ -0,0 +1,138 @@ +name: Cloud database tests +on: + workflow_call: + workflow_dispatch: + inputs: + target: + description: Which managed database to test against + required: true + default: all + type: choice + options: + - all + - azure-sqlserver + - azure-postgresql + - aurora-postgresql + - rds-sqlserver +env: + DOTNET_NOLOGO: true + # Its own region rather than the AWS_REGION secret the SQS tests use: that one is at its VPC + # ceiling, so RDS has no default VPC to place a publicly accessible instance in. us-east-2 already + # has one, has VPC headroom, and offers both instance classes these targets ask for. + AWS_DATABASE_REGION: us-east-2 +defaults: + run: + shell: pwsh +concurrency: + # Deliberately not cancel-in-progress: cancelling a run mid-flight risks skipping the teardown + # step and leaving billable cloud resources behind. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false +jobs: + plan: + name: Plan + runs-on: ubuntu-latest + outputs: + targets: ${{ steps.select.outputs.targets }} + steps: + - name: Select targets + id: select + run: | + $all = @( + @{ target = 'azure-sqlserver'; provider = 'SqlServer' } + @{ target = 'azure-postgresql'; provider = 'PostgreSql' } + @{ target = 'aurora-postgresql'; provider = 'PostgreSql' } + @{ target = 'rds-sqlserver'; provider = 'SqlServer' } + ) + + # A tag push has no input, and runs everything. + $requested = '${{ inputs.target }}' + $targets = if (-not $requested -or $requested -eq 'all') { $all } else { $all | Where-Object { $_.target -eq $requested } } + + if (-not $targets) { + throw "No cloud target named '$requested'." + } + + $targets | ForEach-Object { Write-Output "Will test against $($_.target)" } + + $matrix = @{ include = @($targets) } + "targets=$(ConvertTo-Json -InputObject $matrix -Compress -Depth 4)" | Out-File -FilePath $Env:GITHUB_OUTPUT -Encoding utf8 -Append + + test: + name: ${{ matrix.target }} + needs: plan + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.targets) }} + # The two suites that exercise the persister. Named outright rather than selected by test + # category, because the category also carries the transport tests, which test the queue + # transport against its own connection string and have no business running here. + env: + PERSISTENCE_PROJECT: src/ServiceControl.Persistence.Tests.${{ matrix.provider }}/ServiceControl.Persistence.Tests.${{ matrix.provider }}.csproj + ACCEPTANCE_PROJECT: src/ServiceControl.AcceptanceTests.${{ matrix.provider }}/ServiceControl.AcceptanceTests.${{ matrix.provider }}.csproj + steps: + - name: Check for secrets + env: + SECRETS_AVAILABLE: ${{ secrets.SECRETS_AVAILABLE }} + run: exit $(If ($env:SECRETS_AVAILABLE -eq 'true') { 0 } Else { 1 }) + - name: Checkout + uses: actions/checkout@v7.0.1 + with: + fetch-depth: 0 + - name: Setup .NET SDK + uses: actions/setup-dotnet@v6.0.0 + with: + global-json-file: global.json + - name: Determine runtime version + run: | + # Read settings from Custom.Build.props + [xml]$xml = Get-Content ./src/Custom.Build.props + $runtimeVersion = $xml.selectNodes('/Project/PropertyGroup/RuntimeFrameworkVersion').InnerText + if (-not ($runtimeVersion)) { + throw "Missing RuntimeFrameworkVersion setting in Custom.Build.props" + } + echo "RuntimeVersion=$runtimeVersion" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append + - name: Install .NET Runtime + shell: bash + run: | + ./tools/dotnet-install.sh --skip-non-versioned-files --install-dir /usr/share/dotnet --runtime dotnet --version ${{ env.RuntimeVersion }} + ./tools/dotnet-install.sh --skip-non-versioned-files --install-dir /usr/share/dotnet --runtime aspnetcore --version ${{ env.RuntimeVersion }} + - name: Build + id: build + background: true + run: | + dotnet build $Env:PERSISTENCE_PROJECT --configuration Release + dotnet build $Env:ACCEPTANCE_PROJECT --configuration Release + - name: Azure login + uses: azure/login@v3.0.2 + if: startsWith(matrix.target, 'azure-') + with: + creds: ${{ secrets.AZURE_ACI_CREDENTIALS }} + - name: Setup AWS environment variables + if: startsWith(matrix.target, 'aurora-') || startsWith(matrix.target, 'rds-') + run: | + echo "AWS_REGION=${{ env.AWS_DATABASE_REGION }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append + echo "AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append + echo "AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append + # Unlike ci.yml, which holds its cloud resources back until the build is done, provisioning + # starts first: it is the long pole here at anywhere from 3 to 25 minutes, and the database is + # torn down by this action's post step whatever the job does afterwards. + # + # It has to come after the login steps above. Post steps run in reverse, so this one's teardown + # runs before azure/login's post step signs the CLI out from under it. + - name: Provision database + uses: ./.github/actions/cloud-database + with: + target: ${{ matrix.target }} + # The server is what isolates one run from another, so this has to be unique per run. It + # includes the attempt because run_id does not change when a run is retried, and the + # previous attempt's server may still be there or still be deleting. + name: sc-ct-${{ github.run_id }}-${{ github.run_attempt }} + - name: Wait for build + wait: build + - name: Run tests + run: ./tools/run-tests.ps1 -Projects "$Env:PERSISTENCE_PROJECT`n$Env:ACCEPTANCE_PROJECT" -MaxParallel 2 + env: + ServiceControl_TESTS_FILTER: ${{ matrix.provider }} + PARTICULARSOFTWARE_LICENSE: ${{ secrets.LICENSETEXT }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ac8149db33..e9f2fea97d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,4 +15,7 @@ jobs: secrets: inherit db-container: uses: ./.github/workflows/build-db-container.yml + secrets: inherit + cloud-tests: + uses: ./.github/workflows/cloud-database-tests.yml secrets: inherit \ No newline at end of file