Repository navigation
Benchmark every data type separately, with KeyLoad in each #444
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| # One job per database, data type and node count. Each job starts the database with Aspire, runs the type's | |
| # shared scenario and saves its result file whether or not the run passed. Website then renders the results. | |
| on: | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| permissions: {contents: read} | |
| concurrency: | |
| group: keyload-benchmarks-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' }} | |
| jobs: | |
| image: | |
| name: Resolve the KeyLoad image | |
| if: github.repository == 'managedcode/KeyLoad' && github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 5 | |
| permissions: {contents: read, packages: read} | |
| outputs: | |
| reference: ${{ steps.resolve.outputs.reference }} | |
| steps: | |
| - name: Resolve the latest released KeyLoad server image to one digest | |
| id: resolve | |
| env: | |
| REGISTRY_TOKEN: ${{ github.token }} | |
| run: | | |
| printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin | |
| digest=$(docker buildx imagetools inspect ghcr.io/managedcode/keyload:latest --format '{{json .Manifest}}' | jq -er '.digest') | |
| [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] | |
| printf 'reference=ghcr.io/managedcode/keyload:latest@%s\n' "$digest" >> "$GITHUB_OUTPUT" | |
| docker logout ghcr.io | |
| measure: | |
| name: ${{ matrix.database }} / ${{ matrix.type }} / ${{ matrix.nodes }} node(s) | |
| needs: [image] | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 240 | |
| permissions: {contents: read, packages: read} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - {database: 'KeyLoad', type: documents, nodes: 1} | |
| - {database: 'KeyLoad', type: documents, nodes: 3} | |
| - {database: 'KeyLoad', type: key-value, nodes: 1} | |
| - {database: 'KeyLoad', type: key-value, nodes: 3} | |
| - {database: 'KeyLoad', type: relational, nodes: 1} | |
| - {database: 'KeyLoad', type: relational, nodes: 3} | |
| - {database: 'KeyLoad', type: graph, nodes: 1} | |
| - {database: 'KeyLoad', type: graph, nodes: 3} | |
| - {database: 'KeyLoad', type: vector, nodes: 1} | |
| - {database: 'KeyLoad', type: vector, nodes: 3} | |
| - {database: 'KeyLoad', type: search, nodes: 1} | |
| - {database: 'KeyLoad', type: search, nodes: 3} | |
| - {database: 'KeyLoad', type: time-series, nodes: 1} | |
| - {database: 'KeyLoad', type: time-series, nodes: 3} | |
| - {database: 'KeyLoad', type: analytics, nodes: 1} | |
| - {database: 'KeyLoad', type: analytics, nodes: 3} | |
| - {database: 'KeyLoad', type: queue, nodes: 1} | |
| - {database: 'KeyLoad', type: queue, nodes: 3} | |
| - {database: 'KeyLoad', type: events, nodes: 1} | |
| - {database: 'KeyLoad', type: events, nodes: 3} | |
| - {database: 'MongoDB', type: documents, nodes: 1} | |
| - {database: 'MongoDB', type: documents, nodes: 3} | |
| - {database: 'SurrealDB', type: documents, nodes: 1} | |
| - {database: 'Redis', type: key-value, nodes: 1} | |
| - {database: 'Redis', type: key-value, nodes: 3} | |
| - {database: 'PostgreSQL + pgvector', type: relational, nodes: 1} | |
| - {database: 'PostgreSQL + pgvector', type: relational, nodes: 3} | |
| - {database: 'SQL Server', type: relational, nodes: 1} | |
| - {database: 'MySQL', type: relational, nodes: 1} | |
| - {database: 'Neo4j', type: graph, nodes: 1} | |
| - {database: 'HelixDB', type: graph, nodes: 1} | |
| - {database: 'SurrealDB', type: graph, nodes: 1} | |
| - {database: 'Qdrant', type: vector, nodes: 1} | |
| - {database: 'Qdrant', type: vector, nodes: 3} | |
| - {database: 'PostgreSQL + pgvector', type: vector, nodes: 1} | |
| - {database: 'PostgreSQL + pgvector', type: vector, nodes: 3} | |
| - {database: 'OpenSearch', type: search, nodes: 1} | |
| - {database: 'OpenSearch', type: search, nodes: 3} | |
| - {database: 'TimescaleDB', type: time-series, nodes: 1} | |
| - {database: 'InfluxDB', type: time-series, nodes: 1} | |
| - {database: 'ClickHouse', type: analytics, nodes: 1} | |
| - {database: 'RabbitMQ', type: queue, nodes: 1} | |
| - {database: 'RabbitMQ', type: queue, nodes: 3} | |
| - {database: 'KurrentDB', type: events, nodes: 1} | |
| - {database: 'KurrentDB', type: events, nodes: 3} | |
| env: | |
| Benchmarks__Database: ${{ matrix.database }} | |
| Benchmarks__Type: ${{ matrix.type }} | |
| Benchmarks__Nodes: ${{ matrix.nodes }} | |
| Benchmarks__KeyLoadImage: ${{ needs.image.outputs.reference }} | |
| Benchmarks__Output: ${{ github.workspace }}/results | |
| steps: | |
| - name: Download source code | |
| uses: actions/checkout@v7 | |
| - name: Set up .NET | |
| uses: actions/setup-dotnet@v6 | |
| with: | |
| global-json-file: global.json | |
| - name: Name this cell | |
| id: cell | |
| run: | | |
| case "$Benchmarks__Type" in | |
| documents) cases=DocumentBenchmarks ;; | |
| key-value) cases=KeyValueBenchmarks ;; | |
| relational) cases=RelationalBenchmarks ;; | |
| graph) cases=GraphBenchmarks ;; | |
| vector) cases=VectorBenchmarks ;; | |
| search) cases=SearchBenchmarks ;; | |
| time-series) cases=TimeSeriesBenchmarks ;; | |
| analytics) cases=AnalyticsBenchmarks ;; | |
| queue) cases=QueueBenchmarks ;; | |
| events) cases=EventBenchmarks ;; | |
| *) exit 1 ;; | |
| esac | |
| slug=$(printf '%s' "$Benchmarks__Database" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-|-$//g') | |
| printf 'cases=%s\nartifact=benchmark-%s-%s-n%s\n' "$cases" "$slug" "$Benchmarks__Type" "$Benchmarks__Nodes" >> "$GITHUB_OUTPUT" | |
| - name: Prepare the Docker host | |
| run: | | |
| sudo sysctl -w vm.max_map_count=262144 | |
| docker info --format 'Docker kernel: {{.KernelVersion}}' | |
| - name: Pull the KeyLoad image | |
| if: matrix.database == 'KeyLoad' | |
| env: | |
| REGISTRY_TOKEN: ${{ github.token }} | |
| run: | | |
| printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin | |
| docker pull "$Benchmarks__KeyLoadImage" | |
| docker logout ghcr.io | |
| - name: Build the benchmark client | |
| run: dotnet build benchmarks/KeyLoad.ComparisonHost/KeyLoad.ComparisonHost.csproj --configuration Release -warnaserror | |
| - name: Run the benchmark | |
| run: >- | |
| dotnet benchmarks/KeyLoad.ComparisonHost/bin/Release/net10.0/KeyLoad.ComparisonHost.dll | |
| --output Detailed --treenode-filter "/*/*/${{ steps.cell.outputs.cases }}/*" | |
| --results-directory "$GITHUB_WORKSPACE/results/test-results" --report-trx --timeout 220m | |
| - name: Save the results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: ${{ steps.cell.outputs.artifact }} | |
| path: results/ | |
| if-no-files-found: warn | |
| retention-days: 90 | |
| website: | |
| name: Trigger Website | |
| needs: [measure] | |
| if: always() && !cancelled() && github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 5 | |
| permissions: {actions: write} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - name: Start the Website workflow for this run | |
| run: gh api --method POST "repos/$GITHUB_REPOSITORY/actions/workflows/website.yml/dispatches" -f ref=main -f "inputs[benchmark_run_id]=$GITHUB_RUN_ID" |