Repository navigation
Fix Release test helpers and complete rejected HTTP responses #374
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| on: | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| inputs: | |
| mode: | |
| description: Run harness contracts or the full database suite | |
| type: choice | |
| options: [contracts, full] | |
| default: full | |
| permissions: {contents: read, actions: read} | |
| concurrency: | |
| group: keyload-benchmarks-${{ github.ref }} | |
| cancel-in-progress: false | |
| env: | |
| NUGET_PACKAGES: ${{ github.workspace }}/artifacts/benchmark-runtime/packages | |
| jobs: | |
| comparison-runtime: | |
| name: Prepare comparison runtime | |
| if: github.repository == 'managedcode/KeyLoad' && github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 150 | |
| permissions: {contents: read, actions: read, packages: read} | |
| outputs: | |
| databases: ${{ steps.plan.outputs.database_matrices }} | |
| runtime-artifact-id: ${{ steps.runtime-bundle.outputs.artifact-id }} | |
| keyload-image: ${{ steps.keyload-image.outputs.reference }} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| Configuration: Release | |
| KEYLOAD_COMPARISON_JOB_NAME: Prepare comparison runtime | |
| steps: | |
| - name: Download source code | |
| uses: actions/checkout@v7 | |
| - name: Read the pinned .NET SDK version | |
| id: dotnet-sdk | |
| shell: bash | |
| run: jq -er '"version=" + .sdk.version' global.json >> "$GITHUB_OUTPUT" | |
| - name: Set up .NET | |
| uses: actions/setup-dotnet@v6 | |
| with: | |
| dotnet-version: ${{ steps.dotnet-sdk.outputs.version }} | |
| # KeyLoad cells run the released server image; this job compiles only the comparison harness. | |
| - name: Restore the comparison harness packages | |
| run: dotnet restore benchmarks/KeyLoad.ComparisonHost/KeyLoad.ComparisonHost.csproj | |
| - name: Prepare the pinned Aspire runtime inside the archive inputs | |
| shell: bash | |
| run: | | |
| aspire_version=$(jq -er '."msbuild-sdks"."Aspire.AppHost.Sdk"' global.json) | |
| [[ "$aspire_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] | |
| aspire_rid="linux-${RUNNER_ARCH,,}" | |
| [[ "$aspire_rid" == linux-x64 || "$aspire_rid" == linux-arm64 ]] | |
| aspire_cli="$NUGET_PACKAGES/aspire.cli.$aspire_rid/$aspire_version/tools/net10.0/$aspire_rid" | |
| dotnet dnx --yes "aspire.cli@$aspire_version" -- setup --install-path "$aspire_cli" --non-interactive | |
| test -x "$aspire_cli/bundle/dcp/dcp" | |
| test -x "$aspire_cli/bundle/dashboard/Aspire.Dashboard" | |
| printf 'AspireCliBundlePath=%s/bundle\n' "$aspire_cli" >> "$GITHUB_ENV" | |
| - name: Build the comparison harness | |
| run: dotnet build benchmarks/KeyLoad.ComparisonHost/KeyLoad.ComparisonHost.csproj --no-restore --configuration Release -p:AspireCliInvocationMode=DnxPinned -warnaserror | |
| - name: Check repository rules | |
| run: node scripts/Features/RepositoryGovernance/verify.mjs | |
| - name: Plan the shared database sessions | |
| id: plan | |
| run: node scripts/Features/BenchmarkComparisons/session-plan.mjs --output="$RUNNER_TEMP/session-plan.json" --github-output="$GITHUB_OUTPUT" | |
| - name: Save the canonical session plan | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: comparison-plan | |
| path: ${{ runner.temp }}/session-plan.json | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Check Docker | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| run: docker version | |
| - name: Resolve the latest released KeyLoad server image to one digest | |
| id: keyload-image | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| env: | |
| REGISTRY_TOKEN: ${{ github.token }} | |
| KEYLOAD_IMAGE_REPOSITORY: ghcr.io/managedcode/keyload | |
| run: | | |
| printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin | |
| digest=$(docker buildx imagetools inspect "$KEYLOAD_IMAGE_REPOSITORY:latest" --format '{{json .Manifest}}' | jq -er '.digest') | |
| [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] | |
| reference="$KEYLOAD_IMAGE_REPOSITORY:latest@$digest" | |
| printf 'reference=%s\n' "$reference" >> "$GITHUB_OUTPUT" | |
| printf 'KeyLoad cells use %s\n' "$reference" >> "$GITHUB_STEP_SUMMARY" | |
| docker logout ghcr.io | |
| - name: Pack the compiled benchmark runtime | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| run: node scripts/Features/BenchmarkComparisons/compiled-test-bundle.mjs --pack | |
| - name: Save compiled runtime identities | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: comparison-compiled-runtime-identity | |
| path: | | |
| artifacts/benchmark-runtime-bundle/benchmark-runtime.json | |
| artifacts/benchmark-runtime-bundle/benchmark-runtime-files.json | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Save the compiled benchmark runtime | |
| id: runtime-bundle | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: comparison-compiled-test-bundle | |
| path: artifacts/benchmark-runtime-bundle/* | |
| if-no-files-found: error | |
| compression-level: 0 | |
| retention-days: 90 | |
| - name: Save original preparation diagnostics | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: comparison-runtime-qualification | |
| path: | | |
| TestResults/** | |
| artifacts/qualification/** | |
| artifacts/code-quality/** | |
| ${{ runner.temp }}/keyload-cell-github/** | |
| if-no-files-found: warn | |
| retention-days: 90 | |
| comparison-keyload: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| permissions: {contents: read, actions: read, packages: read} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).keyload }} | |
| env: &database-environment | |
| GH_TOKEN: ${{ github.token }} | |
| KEYLOAD_BENCHMARK_SESSION_ID: ${{ matrix.id }} | |
| KEYLOAD_COMPARISON_JOB_NAME: ${{ matrix.jobName }} | |
| Benchmarks__Target: ${{ matrix.target }} | |
| Benchmarks__NodeCount: ${{ matrix.nodeCount }} | |
| Benchmarks__Session: 'true' | |
| Benchmarks__SessionId: ${{ matrix.id }} | |
| Benchmarks__EvidenceProfile: session-v1 | |
| Benchmarks__KeyLoadImage: ${{ matrix.target == 'KeyLoad' && needs.comparison-runtime.outputs.keyload-image || '' }} | |
| steps: &database-steps | |
| - name: Download source code | |
| uses: actions/checkout@v7 | |
| - name: Prepare the execution database session | |
| id: setup | |
| uses: ./.github/workflows/Features/BenchmarkComparisons/SessionSetup | |
| with: | |
| runtime-artifact-id: ${{ needs.comparison-runtime.outputs.runtime-artifact-id }} | |
| keyload-image: ${{ matrix.target == 'KeyLoad' && needs.comparison-runtime.outputs.keyload-image || '' }} | |
| - name: Test selected database adapter against real infrastructure | |
| if: matrix.nodeCount == 1 && contains(fromJSON('["TimescaleDB","SQL Server","MySQL","ClickHouse","InfluxDB"]'), matrix.target) | |
| run: dotnet benchmarks/KeyLoad.ComparisonHost/bin/Release/net10.0/KeyLoad.ComparisonHost.dll --output Detailed --github-reporter-style full --maximum-parallel-tests 1 --minimum-expected-tests 1 --treenode-filter '/*/*/AdditionalDatabaseTests/*' --results-directory "$KEYLOAD_COMPARISON_EVIDENCE_DIRECTORY/AdapterTestResults" --report-trx --timeout 15m | |
| - name: Run database workload | |
| id: workload | |
| run: dotnet benchmarks/KeyLoad.ComparisonHost/bin/Release/net10.0/KeyLoad.ComparisonHost.dll --output Detailed --github-reporter-style full --maximum-parallel-tests 1 --minimum-expected-tests 1 --treenode-filter '/*/*/BenchmarkSessionTests/*' --results-directory "$KEYLOAD_COMPARISON_EVIDENCE_DIRECTORY/TestResults" --report-trx --timeout 210m | |
| - name: Record benchmark availability | |
| if: ${{ !cancelled() && (steps.workload.outcome == 'success' || steps.workload.outcome == 'failure') }} | |
| env: | |
| KEYLOAD_WORKLOAD_OUTCOME: ${{ steps.workload.outcome }} | |
| run: node scripts/Features/BenchmarkComparisons/session-finalize.mjs | |
| - name: Save benchmark results | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: ${{ matrix.artifactName }} | |
| path: artifacts/comparisons/sessions/workers/${{ matrix.id }}/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: End this job and retain execution diagnostics | |
| if: always() | |
| uses: ./.github/workflows/Features/BenchmarkComparisons/SessionTeardown | |
| with: | |
| artifact-name: comparison-case-qualification-session-${{ matrix.id }} | |
| comparison-postgresql: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).postgresql }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-qdrant: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).qdrant }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-rabbitmq: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).rabbitmq }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-redis: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).redis }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-neo4j: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).neo4j }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-mongodb: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).mongodb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-opensearch: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).opensearch }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-kurrentdb: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).kurrentdb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-surrealdb: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).surrealdb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-helixdb: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).helixdb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-timescaledb: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).timescaledb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-sqlserver: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).sqlserver }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-mysql: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).mysql }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-clickhouse: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).clickhouse }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-influxdb: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).influxdb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-aggregate: | |
| name: Combine benchmark results | |
| needs: [comparison-runtime, comparison-keyload, comparison-postgresql, comparison-qdrant, comparison-rabbitmq, comparison-redis, comparison-neo4j, comparison-mongodb, comparison-opensearch, comparison-kurrentdb, comparison-surrealdb, comparison-helixdb, comparison-timescaledb, comparison-sqlserver, comparison-mysql, comparison-clickhouse, comparison-influxdb] | |
| if: ${{ always() && !cancelled() && (github.event_name == 'push' || inputs.mode == 'full') && needs.comparison-runtime.result == 'success' }} | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 150 | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - name: Download source code | |
| uses: actions/checkout@v7 | |
| - name: Download original authenticated session results | |
| run: | | |
| mkdir -p artifacts/comparisons/sessions | |
| node scripts/Features/BenchmarkComparisons/session-github-collect.mjs --input="$PWD/artifacts/comparisons/sessions/provider" | |
| - name: Check complete session accounting | |
| run: node scripts/Features/BenchmarkComparisons/session-aggregate-cli.mjs --input="$PWD/artifacts/comparisons/sessions/provider" --output="$PWD/artifacts/comparisons/sessions/aggregate" | |
| - name: Save combined benchmark results | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: comparison-session-suite | |
| path: artifacts/comparisons/sessions/aggregate/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Save GitHub result verification | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: comparison-session-provider-evidence | |
| path: | | |
| artifacts/comparisons/sessions/provider/github/** | |
| artifacts/comparisons/sessions/provider/proof.json | |
| if-no-files-found: error | |
| retention-days: 90 | |
| website-trigger: | |
| name: Trigger Website | |
| needs: [comparison-runtime, comparison-keyload, comparison-postgresql, comparison-qdrant, comparison-rabbitmq, comparison-redis, comparison-neo4j, comparison-mongodb, comparison-opensearch, comparison-kurrentdb, comparison-surrealdb, comparison-helixdb, comparison-timescaledb, comparison-sqlserver, comparison-mysql, comparison-clickhouse, comparison-influxdb, comparison-aggregate] | |
| if: >- | |
| always() && !cancelled() && | |
| (github.event_name == 'push' || inputs.mode == 'full') && | |
| needs.comparison-aggregate.result == 'success' && | |
| !contains(needs.*.result, 'failure') && | |
| !contains(needs.*.result, 'cancelled') && | |
| !contains(needs.*.result, 'skipped') && | |
| github.repository == 'managedcode/KeyLoad' && github.event.repository.id == 477801965 && | |
| github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 5 | |
| permissions: {actions: write} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GH_REPO: ${{ github.repository }} | |
| BENCHMARK_RUN_ID: ${{ github.run_id }} | |
| steps: | |
| - name: Start the separate Website workflow | |
| run: timeout 120s gh api --method POST "repos/$GH_REPO/actions/workflows/website.yml/dispatches" -f ref=main -f "inputs[benchmark_run_id]=$BENCHMARK_RUN_ID" |