Repository navigation
Bind session regression operations to the native comparison result #311
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| on: | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| inputs: | |
| mode: | |
| description: Run harness contracts or the full database suite | |
| type: choice | |
| options: [contracts, full] | |
| default: full | |
| permissions: {contents: read, actions: read} | |
| concurrency: | |
| group: keyload-benchmarks-${{ github.ref }} | |
| cancel-in-progress: false | |
| env: | |
| NUGET_PACKAGES: ${{ github.workspace }}/artifacts/benchmark-runtime/packages | |
| jobs: | |
| comparison-images: | |
| name: Build Docker images | |
| if: github.repository == 'managedcode/KeyLoad' && github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| outputs: | |
| databases: ${{ steps.plan.outputs.database_matrices }} | |
| bundle-artifact-id: ${{ steps.images-bundle.outputs.artifact-id }} | |
| runtime-artifact-id: ${{ steps.runtime-bundle.outputs.artifact-id }} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| KEYLOAD_COMPARISON_JOB_NAME: Build Docker images | |
| steps: | |
| - name: Download source code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - name: Set up .NET | |
| uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 | |
| with: | |
| global-json-file: global.json | |
| - name: Restore .NET packages once | |
| run: dotnet restore KeyLoad.slnx | |
| - name: Build the complete solution once | |
| run: dotnet build KeyLoad.slnx --no-restore --configuration Release -p:AspireCliInvocationMode=DnxPinned | |
| - name: Build current Debug analyzer for native formatting | |
| run: dotnet build src/KeyLoad.Analyzers/KeyLoad.Analyzers.csproj --no-restore --configuration Debug | |
| - name: Check code formatting | |
| run: dotnet format KeyLoad.slnx --verify-no-changes --no-restore | |
| - name: Check repository rules | |
| run: node scripts/Features/RepositoryGovernance/verify.mjs | |
| - name: Test benchmark contracts | |
| run: node scripts/Features/TestInfrastructure/run-tests.mjs --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/KeyLoad.UnitTests*/*/*' --KeyLoadTests:ResultsDirectory=TestResults/benchmark-contracts --KeyLoadTests:ReportTrx=true | |
| - name: Test canonical benchmark worker selection | |
| run: node scripts/Features/TestInfrastructure/run-tests.mjs --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/KeyLoad.UnitTests*/(BenchmarkSessionContractTests|SessionCohortContractTests|CompiledBenchmarkRuntimeBundleTests)/*' --KeyLoadTests:ResultsDirectory=TestResults/session-contracts --KeyLoadTests:ReportTrx=true | |
| - name: Test prompt termination on Aspire resource failures | |
| run: node scripts/Features/TestInfrastructure/run-tests.mjs --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/AspireFailure*/*' --KeyLoadTests:ResultsDirectory=TestResults/aspire-failure --KeyLoadTests:ReportTrx=true | |
| - name: Plan the shared database sessions | |
| id: plan | |
| run: node scripts/Features/BenchmarkComparisons/session-plan.mjs --output="$RUNNER_TEMP/session-plan.json" --github-output="$GITHUB_OUTPUT" | |
| - name: Save the canonical session plan | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-plan | |
| path: ${{ runner.temp }}/session-plan.json | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Check Docker | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| run: docker version | |
| - name: Check bounded Docker registry readiness | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| run: node scripts/Features/TestInfrastructure/run-tests.mjs --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/ImageRegistryReadinessTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/registry-readiness | |
| - name: Build KeyLoad server and benchmark Docker images | |
| id: images | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| run: node scripts/Features/BenchmarkComparisons/prepare-images.mjs | |
| - name: Configure Docker image references | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| env: | |
| KEYLOAD_SERVER_IMAGE: ${{ steps.images.outputs.server-image }} | |
| KEYLOAD_RUNNER_IMAGE: ${{ steps.images.outputs.load-generator-image }} | |
| run: | | |
| test -n "$KEYLOAD_SERVER_IMAGE" && test -n "$KEYLOAD_RUNNER_IMAGE" | |
| printf 'KeyLoad__ContainerImages__Server=%s\nBenchmarks__ContainerImages__LoadGenerator=%s\nKEYLOAD_IMAGE_RECEIPT=%s\n' "$KEYLOAD_SERVER_IMAGE" "$KEYLOAD_RUNNER_IMAGE" "$RUNNER_TEMP/keyload-images/image-receipt.json" >> "$GITHUB_ENV" | |
| - name: Check native benchmark preparation contracts | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| uses: ./.github/workflows/Features/BenchmarkComparisons/SessionContracts | |
| - name: Check Docker image export and import | |
| id: roundtrip | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| run: node scripts/Features/TestInfrastructure/run-tests.mjs --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/ImageBundleRealTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/image-roundtrip | |
| - name: Verify GitHub job identity | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| run: node scripts/Features/TestInfrastructure/run-tests.mjs --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/IsolatedGitHubCurrentJobTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/job-identity | |
| - name: Save Docker images | |
| id: images-bundle | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-image-bundle | |
| path: ${{ runner.temp }}/keyload-image-bundle/* | |
| if-no-files-found: error | |
| compression-level: 0 | |
| retention-days: 90 | |
| - name: Pack the compiled benchmark runtime | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| run: node scripts/Features/BenchmarkComparisons/compiled-test-bundle.mjs --pack | |
| - name: Save compiled runtime identities | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-compiled-runtime-identity | |
| path: | | |
| artifacts/benchmark-runtime-bundle/benchmark-runtime.json | |
| artifacts/benchmark-runtime-bundle/benchmark-runtime-files.json | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Save the compiled benchmark runtime | |
| id: runtime-bundle | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-compiled-test-bundle | |
| path: artifacts/benchmark-runtime-bundle/* | |
| if-no-files-found: error | |
| compression-level: 0 | |
| retention-days: 90 | |
| - name: Clean up Docker registry after image failure | |
| if: ${{ always() && steps.roundtrip.outcome != 'success' && (steps.images.outcome == 'success' || steps.images.outcome == 'failure' || steps.images.outcome == 'cancelled') }} | |
| run: node scripts/Features/BenchmarkComparisons/cleanup-images.mjs | |
| - name: Save original preparation diagnostics | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-images-qualification | |
| path: | | |
| TestResults/** | |
| artifacts/qualification/** | |
| artifacts/code-quality/** | |
| ${{ runner.temp }}/keyload-images/** | |
| ${{ runner.temp }}/keyload-images-build/** | |
| ${{ runner.temp }}/keyload-cell-github/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| comparison-keyload: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).keyload }} | |
| env: &database-environment | |
| GH_TOKEN: ${{ github.token }} | |
| KEYLOAD_BENCHMARK_SESSION_ID: ${{ matrix.id }} | |
| KEYLOAD_COMPARISON_JOB_NAME: ${{ matrix.jobName }} | |
| Benchmarks__Target: ${{ matrix.target }} | |
| Benchmarks__NodeCount: ${{ matrix.nodeCount }} | |
| Benchmarks__Session: 'true' | |
| Benchmarks__SessionId: ${{ matrix.id }} | |
| Benchmarks__EvidenceProfile: session-v1 | |
| steps: &database-steps | |
| - name: Download source code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - name: Prepare the native database session | |
| id: images | |
| uses: ./.github/workflows/Features/BenchmarkComparisons/SessionSetup | |
| with: | |
| image-artifact-id: ${{ needs.comparison-images.outputs.bundle-artifact-id }} | |
| runtime-artifact-id: ${{ needs.comparison-images.outputs.runtime-artifact-id }} | |
| - name: Run database workload | |
| id: workload | |
| run: dotnet tests/KeyLoad.ComparisonTests/bin/Release/net10.0/KeyLoad.ComparisonTests.dll --output Detailed --github-reporter-style full --maximum-parallel-tests 1 --minimum-expected-tests 1 --treenode-filter '/*/*/BenchmarkSessionTests/*' --results-directory "$KEYLOAD_COMPARISON_EVIDENCE_DIRECTORY/TestResults" --report-trx --timeout 210m | |
| - name: Record benchmark availability | |
| if: ${{ !cancelled() && (steps.workload.outcome == 'success' || steps.workload.outcome == 'failure') }} | |
| env: | |
| KEYLOAD_WORKLOAD_OUTCOME: ${{ steps.workload.outcome }} | |
| run: node scripts/Features/BenchmarkComparisons/session-finalize.mjs | |
| - name: Save benchmark results | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: ${{ matrix.artifactName }} | |
| path: artifacts/comparisons/sessions/workers/${{ matrix.id }}/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: End this job and retain native diagnostics | |
| if: always() | |
| uses: ./.github/workflows/Features/BenchmarkComparisons/SessionTeardown | |
| with: | |
| setup-outcome: ${{ steps.images.outcome }} | |
| artifact-name: comparison-case-qualification-session-${{ matrix.id }} | |
| comparison-postgresql: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).postgresql }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-qdrant: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).qdrant }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-rabbitmq: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).rabbitmq }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-redis: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).redis }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-neo4j: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).neo4j }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-mongodb: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).mongodb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-opensearch: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).opensearch }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-kurrentdb: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).kurrentdb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-surrealdb: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).surrealdb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-helixdb: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-images.outputs.databases).helixdb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| keyload-functional-heavy-load: | |
| name: KeyLoad functional RF3 million-record ingestion | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 180 | |
| steps: | |
| - name: Download source code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - name: Prepare real functional RF3 ingestion | |
| uses: ./.github/workflows/Features/BenchmarkComparisons/FunctionalHeavyLoadSetup | |
| with: | |
| bundle-artifact-id: ${{ needs.comparison-images.outputs.bundle-artifact-id }} | |
| - name: Test exclusive heavy-load admission | |
| run: node scripts/Features/TestInfrastructure/run-tests.mjs --KeyLoadTests:Suite=rf3 '--KeyLoadTests:Filter=/*/*/HeavyDocumentLoadAdmissionTests/*' --KeyLoadTests:ResultsDirectory=TestResults/rf3-heavy-admission --KeyLoadTests:ReportTrx=true | |
| - name: Test ingestion while SDK and MCP reads are active | |
| run: node scripts/Features/TestInfrastructure/run-tests.mjs --KeyLoadTests:Suite=rf3 '--KeyLoadTests:Filter=/*/*/HeavyDocumentLoadRf3Tests/*' --KeyLoadTests:HeavyLoad:Enabled=true --KeyLoadTests:TimeoutMinutes=140 --KeyLoadTests:ResultsDirectory=TestResults/rf3-heavy-load --KeyLoadTests:ReportTrx=true | |
| - name: Clean up this job's Docker registry | |
| if: always() | |
| run: node scripts/Features/BenchmarkComparisons/cleanup-images.mjs | |
| - name: Save original functional load results and image receipts | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: keyload-functional-heavy-load | |
| path: | | |
| TestResults/rf3-heavy-admission/** | |
| TestResults/rf3-heavy-load/** | |
| ${{ runner.temp }}/keyload-images/** | |
| artifacts/code-quality/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| comparison-aggregate: | |
| name: Combine benchmark results | |
| needs: [comparison-images, comparison-keyload, comparison-postgresql, comparison-qdrant, comparison-rabbitmq, comparison-redis, comparison-neo4j, comparison-mongodb, comparison-opensearch, comparison-kurrentdb, comparison-surrealdb, comparison-helixdb, keyload-functional-heavy-load] | |
| if: ${{ always() && !cancelled() && (github.event_name == 'push' || inputs.mode == 'full') && needs.comparison-images.result == 'success' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - name: Download source code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - name: Download original authenticated session results | |
| run: | | |
| mkdir -p artifacts/comparisons/sessions | |
| node scripts/Features/BenchmarkComparisons/session-github-collect.mjs --input="$PWD/artifacts/comparisons/sessions/provider" | |
| - name: Check complete session accounting | |
| run: node scripts/Features/BenchmarkComparisons/session-aggregate-cli.mjs --input="$PWD/artifacts/comparisons/sessions/provider" --output="$PWD/artifacts/comparisons/sessions/aggregate" | |
| - name: Save combined benchmark results | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-session-suite | |
| path: artifacts/comparisons/sessions/aggregate/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Save GitHub result verification | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-session-provider-evidence | |
| path: | | |
| artifacts/comparisons/sessions/provider/github/** | |
| artifacts/comparisons/sessions/provider/proof.json | |
| if-no-files-found: error | |
| retention-days: 90 | |
| website-trigger: | |
| name: Trigger Website | |
| needs: [comparison-images, comparison-keyload, comparison-postgresql, comparison-qdrant, comparison-rabbitmq, comparison-redis, comparison-neo4j, comparison-mongodb, comparison-opensearch, comparison-kurrentdb, comparison-surrealdb, comparison-helixdb, keyload-functional-heavy-load, comparison-aggregate] | |
| if: >- | |
| always() && !cancelled() && | |
| (github.event_name == 'push' || inputs.mode == 'full') && | |
| needs.comparison-aggregate.result == 'success' && | |
| !contains(needs.*.result, 'failure') && | |
| !contains(needs.*.result, 'cancelled') && | |
| !contains(needs.*.result, 'skipped') && | |
| github.repository == 'managedcode/KeyLoad' && github.event.repository.id == 477801965 && | |
| github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| permissions: {actions: write} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GH_REPO: ${{ github.repository }} | |
| BENCHMARK_RUN_ID: ${{ github.run_id }} | |
| steps: | |
| - name: Start the separate Website workflow | |
| run: timeout 120s gh api --method POST "repos/$GH_REPO/actions/workflows/website.yml/dispatches" -f ref=main -f "inputs[benchmark_run_id]=$BENCHMARK_RUN_ID" |