Repository navigation
Add native SQL and analytical benchmark targets #345
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| on: | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| inputs: | |
| mode: | |
| description: Run harness contracts or the full database suite | |
| type: choice | |
| options: [contracts, full] | |
| default: full | |
| permissions: {contents: read, actions: read} | |
| concurrency: | |
| group: keyload-benchmarks-${{ github.ref }} | |
| cancel-in-progress: false | |
| env: | |
| NUGET_PACKAGES: ${{ github.workspace }}/artifacts/benchmark-runtime/packages | |
| jobs: | |
| comparison-runtime: | |
| name: Prepare comparison runtime | |
| if: github.repository == 'managedcode/KeyLoad' && github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 150 | |
| permissions: {contents: read, actions: read, packages: read} | |
| outputs: | |
| databases: ${{ steps.plan.outputs.database_matrices }} | |
| runtime-artifact-id: ${{ steps.runtime-bundle.outputs.artifact-id }} | |
| keyload-image: ${{ steps.keyload-image.outputs.reference }} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| Configuration: Release | |
| KEYLOAD_COMPARISON_JOB_NAME: Prepare comparison runtime | |
| steps: | |
| - name: Download source code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - name: Set up .NET | |
| uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 | |
| with: | |
| global-json-file: global.json | |
| - name: Restore .NET packages once | |
| run: dotnet restore KeyLoad.slnx | |
| - name: Prepare the pinned Aspire runtime inside the archive inputs | |
| shell: bash | |
| run: | | |
| aspire_version=$(jq -er '."msbuild-sdks"."Aspire.AppHost.Sdk"' global.json) | |
| [[ "$aspire_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] | |
| aspire_rid="linux-${RUNNER_ARCH,,}" | |
| [[ "$aspire_rid" == linux-x64 || "$aspire_rid" == linux-arm64 ]] | |
| aspire_cli="$NUGET_PACKAGES/aspire.cli.$aspire_rid/$aspire_version/tools/net10.0/$aspire_rid" | |
| dotnet dnx --yes "aspire.cli@$aspire_version" -- setup --install-path "$aspire_cli" --non-interactive | |
| test -x "$aspire_cli/bundle/dcp/dcp" | |
| test -x "$aspire_cli/bundle/dashboard/Aspire.Dashboard" | |
| printf 'AspireCliBundlePath=%s/bundle\n' "$aspire_cli" >> "$GITHUB_ENV" | |
| - name: Build the complete solution once | |
| run: dotnet build KeyLoad.slnx --no-restore --configuration Release -p:AspireCliInvocationMode=DnxPinned -warnaserror | |
| - name: Build current Debug analyzer for execution formatting | |
| run: dotnet build analyzers/KeyLoad.Analyzers/KeyLoad.Analyzers.csproj --no-restore --configuration Debug -warnaserror | |
| - name: Check code formatting | |
| run: dotnet format KeyLoad.slnx --verify-no-changes --no-restore | |
| - name: Check repository rules | |
| run: node scripts/Features/RepositoryGovernance/verify.mjs | |
| - name: Plan the shared database sessions | |
| id: plan | |
| run: node scripts/Features/BenchmarkComparisons/session-plan.mjs --output="$RUNNER_TEMP/session-plan.json" --github-output="$GITHUB_OUTPUT" | |
| - name: Save the canonical session plan | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-plan | |
| path: ${{ runner.temp }}/session-plan.json | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Check Docker | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| run: docker version | |
| - name: Resolve the latest released KeyLoad server image to one digest | |
| id: keyload-image | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| env: | |
| REGISTRY_TOKEN: ${{ github.token }} | |
| KEYLOAD_IMAGE_REPOSITORY: ghcr.io/managedcode/keyload | |
| run: | | |
| printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin | |
| digest=$(docker buildx imagetools inspect "$KEYLOAD_IMAGE_REPOSITORY:latest" --format '{{json .Manifest}}' | jq -er '.digest') | |
| [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] | |
| reference="$KEYLOAD_IMAGE_REPOSITORY:latest@$digest" | |
| printf 'reference=%s\n' "$reference" >> "$GITHUB_OUTPUT" | |
| printf 'KeyLoad cells use `%s`\n' "$reference" >> "$GITHUB_STEP_SUMMARY" | |
| docker logout ghcr.io | |
| - name: Pack the compiled benchmark runtime | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| run: node scripts/Features/BenchmarkComparisons/compiled-test-bundle.mjs --pack | |
| - name: Save compiled runtime identities | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-compiled-runtime-identity | |
| path: | | |
| artifacts/benchmark-runtime-bundle/benchmark-runtime.json | |
| artifacts/benchmark-runtime-bundle/benchmark-runtime-files.json | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Save the compiled benchmark runtime | |
| id: runtime-bundle | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-compiled-test-bundle | |
| path: artifacts/benchmark-runtime-bundle/* | |
| if-no-files-found: error | |
| compression-level: 0 | |
| retention-days: 90 | |
| - name: Save original preparation diagnostics | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-runtime-qualification | |
| path: | | |
| TestResults/** | |
| artifacts/qualification/** | |
| artifacts/code-quality/** | |
| ${{ runner.temp }}/keyload-cell-github/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| comparison-keyload: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| permissions: {contents: read, actions: read, packages: read} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).keyload }} | |
| env: &database-environment | |
| GH_TOKEN: ${{ github.token }} | |
| KEYLOAD_BENCHMARK_SESSION_ID: ${{ matrix.id }} | |
| KEYLOAD_COMPARISON_JOB_NAME: ${{ matrix.jobName }} | |
| Benchmarks__Target: ${{ matrix.target }} | |
| Benchmarks__NodeCount: ${{ matrix.nodeCount }} | |
| Benchmarks__Session: 'true' | |
| Benchmarks__SessionId: ${{ matrix.id }} | |
| Benchmarks__EvidenceProfile: session-v1 | |
| Benchmarks__KeyLoadImage: ${{ matrix.target == 'KeyLoad' && needs.comparison-runtime.outputs.keyload-image || '' }} | |
| steps: &database-steps | |
| - name: Download source code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - name: Prepare the execution database session | |
| id: setup | |
| uses: ./.github/workflows/Features/BenchmarkComparisons/SessionSetup | |
| with: | |
| runtime-artifact-id: ${{ needs.comparison-runtime.outputs.runtime-artifact-id }} | |
| keyload-image: ${{ matrix.target == 'KeyLoad' && needs.comparison-runtime.outputs.keyload-image || '' }} | |
| - name: Test selected database adapter against real infrastructure | |
| if: matrix.nodeCount == 1 && contains(fromJSON('["TimescaleDB","SQL Server","MySQL","ClickHouse","InfluxDB"]'), matrix.target) | |
| run: dotnet benchmarks/KeyLoad.ComparisonHost/bin/Release/net10.0/KeyLoad.ComparisonHost.dll --output Detailed --github-reporter-style full --maximum-parallel-tests 1 --minimum-expected-tests 1 --treenode-filter '/*/*/AdditionalDatabaseTests/*' --results-directory "$KEYLOAD_COMPARISON_EVIDENCE_DIRECTORY/AdapterTestResults" --report-trx --timeout 15m | |
| - name: Run database workload | |
| id: workload | |
| run: dotnet benchmarks/KeyLoad.ComparisonHost/bin/Release/net10.0/KeyLoad.ComparisonHost.dll --output Detailed --github-reporter-style full --maximum-parallel-tests 1 --minimum-expected-tests 1 --treenode-filter '/*/*/BenchmarkSessionTests/*' --results-directory "$KEYLOAD_COMPARISON_EVIDENCE_DIRECTORY/TestResults" --report-trx --timeout 210m | |
| - name: Record benchmark availability | |
| if: ${{ !cancelled() && (steps.workload.outcome == 'success' || steps.workload.outcome == 'failure') }} | |
| env: | |
| KEYLOAD_WORKLOAD_OUTCOME: ${{ steps.workload.outcome }} | |
| run: node scripts/Features/BenchmarkComparisons/session-finalize.mjs | |
| - name: Save benchmark results | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: ${{ matrix.artifactName }} | |
| path: artifacts/comparisons/sessions/workers/${{ matrix.id }}/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: End this job and retain execution diagnostics | |
| if: always() | |
| uses: ./.github/workflows/Features/BenchmarkComparisons/SessionTeardown | |
| with: | |
| artifact-name: comparison-case-qualification-session-${{ matrix.id }} | |
| comparison-postgresql: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).postgresql }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-qdrant: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).qdrant }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-rabbitmq: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).rabbitmq }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-redis: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).redis }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-neo4j: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).neo4j }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-mongodb: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).mongodb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-opensearch: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).opensearch }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-kurrentdb: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).kurrentdb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-surrealdb: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).surrealdb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-helixdb: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).helixdb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-timescaledb: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).timescaledb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-sqlserver: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).sqlserver }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-mysql: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).mysql }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-clickhouse: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).clickhouse }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-influxdb: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).influxdb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| keyload-functional-heavy-load: | |
| name: KeyLoad functional CLUSTER million-record ingestion | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 180 | |
| steps: | |
| - name: Download source code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - name: Prepare real functional CLUSTER ingestion | |
| uses: ./.github/workflows/Features/BenchmarkComparisons/FunctionalHeavyLoadSetup | |
| - name: Test exclusive heavy-load admission | |
| run: node scripts/Features/TestInfrastructure/run-tests.mjs --KeyLoadTests:Suite=cluster '--KeyLoadTests:Filter=/*/*/HeavyDocumentLoadAdmissionTests/*' --KeyLoadTests:ResultsDirectory=TestResults/cluster-heavy-admission --KeyLoadTests:ReportTrx=true | |
| - name: Test ingestion while SDK and MCP reads are active | |
| run: node scripts/Features/TestInfrastructure/run-tests.mjs --KeyLoadTests:Suite=cluster '--KeyLoadTests:Filter=/*/*/HeavyDocumentLoadClusterTests/*' --KeyLoadTests:HeavyLoad:Enabled=true --KeyLoadTests:TimeoutMinutes=140 --KeyLoadTests:ResultsDirectory=TestResults/cluster-heavy-load --KeyLoadTests:ReportTrx=true | |
| - name: Save original functional load results and execution process receipts | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: keyload-functional-heavy-load | |
| path: | | |
| TestResults/cluster-heavy-admission/** | |
| TestResults/cluster-heavy-load/** | |
| artifacts/code-quality/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| comparison-aggregate: | |
| name: Combine benchmark results | |
| needs: [comparison-runtime, comparison-keyload, comparison-postgresql, comparison-qdrant, comparison-rabbitmq, comparison-redis, comparison-neo4j, comparison-mongodb, comparison-opensearch, comparison-kurrentdb, comparison-surrealdb, comparison-helixdb, comparison-timescaledb, comparison-sqlserver, comparison-mysql, comparison-clickhouse, comparison-influxdb, keyload-functional-heavy-load] | |
| if: ${{ always() && !cancelled() && (github.event_name == 'push' || inputs.mode == 'full') && needs.comparison-runtime.result == 'success' }} | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 150 | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - name: Download source code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - name: Download original authenticated session results | |
| run: | | |
| mkdir -p artifacts/comparisons/sessions | |
| node scripts/Features/BenchmarkComparisons/session-github-collect.mjs --input="$PWD/artifacts/comparisons/sessions/provider" | |
| - name: Check complete session accounting | |
| run: node scripts/Features/BenchmarkComparisons/session-aggregate-cli.mjs --input="$PWD/artifacts/comparisons/sessions/provider" --output="$PWD/artifacts/comparisons/sessions/aggregate" | |
| - name: Save combined benchmark results | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-session-suite | |
| path: artifacts/comparisons/sessions/aggregate/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Save GitHub result verification | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-session-provider-evidence | |
| path: | | |
| artifacts/comparisons/sessions/provider/github/** | |
| artifacts/comparisons/sessions/provider/proof.json | |
| if-no-files-found: error | |
| retention-days: 90 | |
| website-trigger: | |
| name: Trigger Website | |
| needs: [comparison-runtime, comparison-keyload, comparison-postgresql, comparison-qdrant, comparison-rabbitmq, comparison-redis, comparison-neo4j, comparison-mongodb, comparison-opensearch, comparison-kurrentdb, comparison-surrealdb, comparison-helixdb, comparison-timescaledb, comparison-sqlserver, comparison-mysql, comparison-clickhouse, comparison-influxdb, keyload-functional-heavy-load, comparison-aggregate] | |
| if: >- | |
| always() && !cancelled() && | |
| (github.event_name == 'push' || inputs.mode == 'full') && | |
| needs.comparison-aggregate.result == 'success' && | |
| !contains(needs.*.result, 'failure') && | |
| !contains(needs.*.result, 'cancelled') && | |
| !contains(needs.*.result, 'skipped') && | |
| github.repository == 'managedcode/KeyLoad' && github.event.repository.id == 477801965 && | |
| github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 5 | |
| permissions: {actions: write} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GH_REPO: ${{ github.repository }} | |
| BENCHMARK_RUN_ID: ${{ github.run_id }} | |
| steps: | |
| - name: Start the separate Website workflow | |
| run: timeout 120s gh api --method POST "repos/$GH_REPO/actions/workflows/website.yml/dispatches" -f ref=main -f "inputs[benchmark_run_id]=$BENCHMARK_RUN_ID" |