KeyLoad/.github/workflows/benchmarks.yml at main · managedcode/KeyLoad · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
name: Benchmarks
# One job per database, data type and node count. Each job starts the database with Aspire, runs the type's
# shared scenario and saves its result file whether or not the run passed. Website then renders the results.
on:
  push:
    branches: [main]
  workflow_dispatch:
permissions: {contents: read}
concurrency:
  group: keyload-benchmarks-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' }}
jobs:
  image:
    name: Resolve the KeyLoad image
    if: github.repository == 'managedcode/KeyLoad' && github.ref == 'refs/heads/main'
    runs-on: ubuntu-22.04
    timeout-minutes: 5
    permissions: {contents: read, packages: read}
    outputs:
      reference: ${{ steps.resolve.outputs.reference }}
    steps:
      - name: Resolve the latest released KeyLoad server image to one digest
        id: resolve
        env:
          REGISTRY_TOKEN: ${{ github.token }}
        run: |
          printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin
          digest=$(docker buildx imagetools inspect ghcr.io/managedcode/keyload:latest --format '{{json .Manifest}}' | jq -er '.digest')
          [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]
          printf 'reference=ghcr.io/managedcode/keyload:latest@%s\n' "$digest" >> "$GITHUB_OUTPUT"
          docker logout ghcr.io
  measure:
    name: ${{ matrix.database }} / ${{ matrix.type }} / ${{ matrix.nodes }} node(s)
    needs: [image]
    runs-on: ubuntu-22.04
    timeout-minutes: 240
    permissions: {contents: read, packages: read}
    strategy:
      fail-fast: false
      matrix:
        include:
          - {database: 'KeyLoad', type: documents, nodes: 1}
          - {database: 'KeyLoad', type: documents, nodes: 3}
          - {database: 'KeyLoad', type: key-value, nodes: 1}
          - {database: 'KeyLoad', type: key-value, nodes: 3}
          - {database: 'KeyLoad', type: relational, nodes: 1}
          - {database: 'KeyLoad', type: relational, nodes: 3}
          - {database: 'KeyLoad', type: graph, nodes: 1}
          - {database: 'KeyLoad', type: graph, nodes: 3}
          - {database: 'KeyLoad', type: vector, nodes: 1}
          - {database: 'KeyLoad', type: vector, nodes: 3}
          - {database: 'KeyLoad', type: search, nodes: 1}
          - {database: 'KeyLoad', type: search, nodes: 3}
          - {database: 'KeyLoad', type: time-series, nodes: 1}
          - {database: 'KeyLoad', type: time-series, nodes: 3}
          - {database: 'KeyLoad', type: analytics, nodes: 1}
          - {database: 'KeyLoad', type: analytics, nodes: 3}
          - {database: 'KeyLoad', type: queue, nodes: 1}
          - {database: 'KeyLoad', type: queue, nodes: 3}
          - {database: 'KeyLoad', type: events, nodes: 1}
          - {database: 'KeyLoad', type: events, nodes: 3}
          - {database: 'MongoDB', type: documents, nodes: 1}
          - {database: 'MongoDB', type: documents, nodes: 3}
          - {database: 'SurrealDB', type: documents, nodes: 1}
          - {database: 'Redis', type: key-value, nodes: 1}
          - {database: 'Redis', type: key-value, nodes: 3}
          - {database: 'PostgreSQL + pgvector', type: relational, nodes: 1}
          - {database: 'PostgreSQL + pgvector', type: relational, nodes: 3}
          - {database: 'SQL Server', type: relational, nodes: 1}
          - {database: 'MySQL', type: relational, nodes: 1}
          - {database: 'Neo4j', type: graph, nodes: 1}
          - {database: 'HelixDB', type: graph, nodes: 1}
          - {database: 'SurrealDB', type: graph, nodes: 1}
          - {database: 'Qdrant', type: vector, nodes: 1}
          - {database: 'Qdrant', type: vector, nodes: 3}
          - {database: 'PostgreSQL + pgvector', type: vector, nodes: 1}
          - {database: 'PostgreSQL + pgvector', type: vector, nodes: 3}
          - {database: 'OpenSearch', type: search, nodes: 1}
          - {database: 'OpenSearch', type: search, nodes: 3}
          - {database: 'TimescaleDB', type: time-series, nodes: 1}
          - {database: 'InfluxDB', type: time-series, nodes: 1}
          - {database: 'ClickHouse', type: analytics, nodes: 1}
          - {database: 'RabbitMQ', type: queue, nodes: 1}
          - {database: 'RabbitMQ', type: queue, nodes: 3}
          - {database: 'KurrentDB', type: events, nodes: 1}
          - {database: 'KurrentDB', type: events, nodes: 3}
    env:
      Benchmarks__Database: ${{ matrix.database }}
      Benchmarks__Type: ${{ matrix.type }}
      Benchmarks__Nodes: ${{ matrix.nodes }}
      Benchmarks__KeyLoadImage: ${{ needs.image.outputs.reference }}
      Benchmarks__Output: ${{ github.workspace }}/results
    steps:
      - name: Download source code
        uses: actions/checkout@v7
      - name: Set up .NET
        uses: actions/setup-dotnet@v6
        with:
          global-json-file: global.json
      - name: Name this cell
        id: cell
        run: |
          case "$Benchmarks__Type" in
            documents) cases=DocumentBenchmarks ;;
            key-value) cases=KeyValueBenchmarks ;;
            relational) cases=RelationalBenchmarks ;;
            graph) cases=GraphBenchmarks ;;
            vector) cases=VectorBenchmarks ;;
            search) cases=SearchBenchmarks ;;
            time-series) cases=TimeSeriesBenchmarks ;;
            analytics) cases=AnalyticsBenchmarks ;;
            queue) cases=QueueBenchmarks ;;
            events) cases=EventBenchmarks ;;
            *) exit 1 ;;
          esac
          slug=$(printf '%s' "$Benchmarks__Database" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-|-$//g')
          printf 'cases=%s\nartifact=benchmark-%s-%s-n%s\n' "$cases" "$slug" "$Benchmarks__Type" "$Benchmarks__Nodes" >> "$GITHUB_OUTPUT"
      - name: Prepare the Docker host
        run: |
          sudo sysctl -w vm.max_map_count=262144
          docker info --format 'Docker kernel: {{.KernelVersion}}'
      - name: Pull the KeyLoad image
        if: matrix.database == 'KeyLoad'
        env:
          REGISTRY_TOKEN: ${{ github.token }}
        run: |
          printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin
          docker pull "$Benchmarks__KeyLoadImage"
          docker logout ghcr.io
      - name: Build the benchmark client
        run: dotnet build benchmarks/KeyLoad.ComparisonHost/KeyLoad.ComparisonHost.csproj --configuration Release -warnaserror
      - name: Run the benchmark
        run: >-
          dotnet benchmarks/KeyLoad.ComparisonHost/bin/Release/net10.0/KeyLoad.ComparisonHost.dll
          --output Detailed --treenode-filter "/*/*/${{ steps.cell.outputs.cases }}/*"
          --results-directory "$GITHUB_WORKSPACE/results/test-results" --report-trx --timeout 220m
      - name: Save the results
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: ${{ steps.cell.outputs.artifact }}
          path: results/
          if-no-files-found: warn
          retention-days: 90
  website:
    name: Trigger Website
    needs: [measure]
    if: always() && !cancelled() && github.ref == 'refs/heads/main'
    runs-on: ubuntu-22.04
    timeout-minutes: 5
    permissions: {actions: write}
    env:
      GH_TOKEN: ${{ github.token }}
    steps:
      - name: Start the Website workflow for this run
        run: gh api --method POST "repos/$GITHUB_REPOSITORY/actions/workflows/website.yml/dispatches" -f ref=main -f "inputs[benchmark_run_id]=$GITHUB_RUN_ID"