Repository navigation
Keep Chrome IPC short and join idle website listener shutdown #336
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| on: | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| inputs: | |
| mode: | |
| description: Run harness contracts or the full database suite | |
| type: choice | |
| options: [contracts, full] | |
| default: full | |
| permissions: {contents: read, actions: read} | |
| concurrency: | |
| group: keyload-benchmarks-${{ github.ref }} | |
| cancel-in-progress: false | |
| env: | |
| NUGET_PACKAGES: ${{ github.workspace }}/artifacts/benchmark-runtime/packages | |
| jobs: | |
| comparison-runtime: | |
| name: Prepare comparison runtime | |
| if: github.repository == 'managedcode/KeyLoad' && github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| outputs: | |
| databases: ${{ steps.plan.outputs.database_matrices }} | |
| runtime-artifact-id: ${{ steps.runtime-bundle.outputs.artifact-id }} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| Configuration: Release | |
| KEYLOAD_COMPARISON_JOB_NAME: Prepare comparison runtime | |
| steps: | |
| - name: Download source code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - name: Set up .NET | |
| uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 | |
| with: | |
| global-json-file: global.json | |
| - name: Restore .NET packages once | |
| run: dotnet restore KeyLoad.slnx | |
| - name: Prepare the pinned Aspire runtime inside the archive inputs | |
| shell: bash | |
| run: | | |
| aspire_version=$(jq -er '."msbuild-sdks"."Aspire.AppHost.Sdk"' global.json) | |
| [[ "$aspire_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] | |
| aspire_rid="linux-${RUNNER_ARCH,,}" | |
| [[ "$aspire_rid" == linux-x64 || "$aspire_rid" == linux-arm64 ]] | |
| aspire_cli="$NUGET_PACKAGES/aspire.cli.$aspire_rid/$aspire_version/tools/net10.0/$aspire_rid" | |
| dotnet dnx --yes "aspire.cli@$aspire_version" -- setup --install-path "$aspire_cli" --non-interactive | |
| test -x "$aspire_cli/bundle/dcp/dcp" | |
| test -x "$aspire_cli/bundle/dashboard/Aspire.Dashboard" | |
| printf 'AspireCliBundlePath=%s/bundle\n' "$aspire_cli" >> "$GITHUB_ENV" | |
| - name: Build the complete solution once | |
| run: dotnet build KeyLoad.slnx --no-restore --configuration Release -p:AspireCliInvocationMode=DnxPinned -warnaserror | |
| - name: Build current Debug analyzer for execution formatting | |
| run: dotnet build analyzers/KeyLoad.Analyzers/KeyLoad.Analyzers.csproj --no-restore --configuration Debug -warnaserror | |
| - name: Check code formatting | |
| run: dotnet format KeyLoad.slnx --verify-no-changes --no-restore | |
| - name: Check repository rules | |
| run: node scripts/Features/RepositoryGovernance/verify.mjs | |
| - name: Plan the shared database sessions | |
| id: plan | |
| run: node scripts/Features/BenchmarkComparisons/session-plan.mjs --output="$RUNNER_TEMP/session-plan.json" --github-output="$GITHUB_OUTPUT" | |
| - name: Save the canonical session plan | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-plan | |
| path: ${{ runner.temp }}/session-plan.json | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Check Docker | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| run: docker version | |
| - name: Pack the compiled benchmark runtime | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| run: node scripts/Features/BenchmarkComparisons/compiled-test-bundle.mjs --pack | |
| - name: Save compiled runtime identities | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-compiled-runtime-identity | |
| path: | | |
| artifacts/benchmark-runtime-bundle/benchmark-runtime.json | |
| artifacts/benchmark-runtime-bundle/benchmark-runtime-files.json | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Save the compiled benchmark runtime | |
| id: runtime-bundle | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-compiled-test-bundle | |
| path: artifacts/benchmark-runtime-bundle/* | |
| if-no-files-found: error | |
| compression-level: 0 | |
| retention-days: 90 | |
| - name: Save original preparation diagnostics | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-runtime-qualification | |
| path: | | |
| TestResults/** | |
| artifacts/qualification/** | |
| artifacts/code-quality/** | |
| ${{ runner.temp }}/keyload-cell-github/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| comparison-keyload: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).keyload }} | |
| env: &database-environment | |
| GH_TOKEN: ${{ github.token }} | |
| KEYLOAD_BENCHMARK_SESSION_ID: ${{ matrix.id }} | |
| KEYLOAD_COMPARISON_JOB_NAME: ${{ matrix.jobName }} | |
| Benchmarks__Target: ${{ matrix.target }} | |
| Benchmarks__NodeCount: ${{ matrix.nodeCount }} | |
| Benchmarks__Session: 'true' | |
| Benchmarks__SessionId: ${{ matrix.id }} | |
| Benchmarks__EvidenceProfile: session-v1 | |
| steps: &database-steps | |
| - name: Download source code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - name: Prepare the execution database session | |
| id: setup | |
| uses: ./.github/workflows/Features/BenchmarkComparisons/SessionSetup | |
| with: | |
| runtime-artifact-id: ${{ needs.comparison-runtime.outputs.runtime-artifact-id }} | |
| - name: Run database workload | |
| id: workload | |
| run: dotnet benchmarks/KeyLoad.ComparisonHost/bin/Release/net10.0/KeyLoad.ComparisonHost.dll --output Detailed --github-reporter-style full --maximum-parallel-tests 1 --minimum-expected-tests 1 --treenode-filter '/*/*/BenchmarkSessionTests/*' --results-directory "$KEYLOAD_COMPARISON_EVIDENCE_DIRECTORY/TestResults" --report-trx --timeout 210m | |
| - name: Record benchmark availability | |
| if: ${{ !cancelled() && (steps.workload.outcome == 'success' || steps.workload.outcome == 'failure') }} | |
| env: | |
| KEYLOAD_WORKLOAD_OUTCOME: ${{ steps.workload.outcome }} | |
| run: node scripts/Features/BenchmarkComparisons/session-finalize.mjs | |
| - name: Save benchmark results | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: ${{ matrix.artifactName }} | |
| path: artifacts/comparisons/sessions/workers/${{ matrix.id }}/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: End this job and retain execution diagnostics | |
| if: always() | |
| uses: ./.github/workflows/Features/BenchmarkComparisons/SessionTeardown | |
| with: | |
| artifact-name: comparison-case-qualification-session-${{ matrix.id }} | |
| comparison-postgresql: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).postgresql }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-qdrant: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).qdrant }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-rabbitmq: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).rabbitmq }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-redis: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).redis }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-neo4j: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).neo4j }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-mongodb: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).mongodb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-opensearch: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).opensearch }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-kurrentdb: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).kurrentdb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-surrealdb: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).surrealdb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-helixdb: | |
| name: ${{ matrix.jobName }} | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).helixdb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| keyload-functional-heavy-load: | |
| name: KeyLoad functional CLUSTER million-record ingestion | |
| if: (github.event_name == 'push' || inputs.mode == 'full') | |
| needs: [comparison-runtime] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 180 | |
| steps: | |
| - name: Download source code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - name: Prepare real functional CLUSTER ingestion | |
| uses: ./.github/workflows/Features/BenchmarkComparisons/FunctionalHeavyLoadSetup | |
| - name: Test exclusive heavy-load admission | |
| run: node scripts/Features/TestInfrastructure/run-tests.mjs --KeyLoadTests:Suite=cluster '--KeyLoadTests:Filter=/*/*/HeavyDocumentLoadAdmissionTests/*' --KeyLoadTests:ResultsDirectory=TestResults/cluster-heavy-admission --KeyLoadTests:ReportTrx=true | |
| - name: Test ingestion while SDK and MCP reads are active | |
| run: node scripts/Features/TestInfrastructure/run-tests.mjs --KeyLoadTests:Suite=cluster '--KeyLoadTests:Filter=/*/*/HeavyDocumentLoadClusterTests/*' --KeyLoadTests:HeavyLoad:Enabled=true --KeyLoadTests:TimeoutMinutes=140 --KeyLoadTests:ResultsDirectory=TestResults/cluster-heavy-load --KeyLoadTests:ReportTrx=true | |
| - name: Save original functional load results and execution process receipts | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: keyload-functional-heavy-load | |
| path: | | |
| TestResults/cluster-heavy-admission/** | |
| TestResults/cluster-heavy-load/** | |
| artifacts/code-quality/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| comparison-aggregate: | |
| name: Combine benchmark results | |
| needs: [comparison-runtime, comparison-keyload, comparison-postgresql, comparison-qdrant, comparison-rabbitmq, comparison-redis, comparison-neo4j, comparison-mongodb, comparison-opensearch, comparison-kurrentdb, comparison-surrealdb, comparison-helixdb, keyload-functional-heavy-load] | |
| if: ${{ always() && !cancelled() && (github.event_name == 'push' || inputs.mode == 'full') && needs.comparison-runtime.result == 'success' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - name: Download source code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - name: Download original authenticated session results | |
| run: | | |
| mkdir -p artifacts/comparisons/sessions | |
| node scripts/Features/BenchmarkComparisons/session-github-collect.mjs --input="$PWD/artifacts/comparisons/sessions/provider" | |
| - name: Check complete session accounting | |
| run: node scripts/Features/BenchmarkComparisons/session-aggregate-cli.mjs --input="$PWD/artifacts/comparisons/sessions/provider" --output="$PWD/artifacts/comparisons/sessions/aggregate" | |
| - name: Save combined benchmark results | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-session-suite | |
| path: artifacts/comparisons/sessions/aggregate/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Save GitHub result verification | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-session-provider-evidence | |
| path: | | |
| artifacts/comparisons/sessions/provider/github/** | |
| artifacts/comparisons/sessions/provider/proof.json | |
| if-no-files-found: error | |
| retention-days: 90 | |
| website-trigger: | |
| name: Trigger Website | |
| needs: [comparison-runtime, comparison-keyload, comparison-postgresql, comparison-qdrant, comparison-rabbitmq, comparison-redis, comparison-neo4j, comparison-mongodb, comparison-opensearch, comparison-kurrentdb, comparison-surrealdb, comparison-helixdb, keyload-functional-heavy-load, comparison-aggregate] | |
| if: >- | |
| always() && !cancelled() && | |
| (github.event_name == 'push' || inputs.mode == 'full') && | |
| needs.comparison-aggregate.result == 'success' && | |
| !contains(needs.*.result, 'failure') && | |
| !contains(needs.*.result, 'cancelled') && | |
| !contains(needs.*.result, 'skipped') && | |
| github.repository == 'managedcode/KeyLoad' && github.event.repository.id == 477801965 && | |
| github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| permissions: {actions: write} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GH_REPO: ${{ github.repository }} | |
| BENCHMARK_RUN_ID: ${{ github.run_id }} | |
| steps: | |
| - name: Start the separate Website workflow | |
| run: timeout 120s gh api --method POST "repos/$GH_REPO/actions/workflows/website.yml/dispatches" -f ref=main -f "inputs[benchmark_run_id]=$BENCHMARK_RUN_ID" |