Disk Benchmarks (A/A) #109
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (c) Microsoft Corporation. All rights reserved. | |
| # Licensed under the MIT license. | |
| # DiskANN Daily A/A Benchmark Stability Test | |
| # | |
| # Runs main vs main at 9 AM UTC every day to detect environment noise. | |
| # If any threshold is breached, a GitHub issue is created to notify @microsoft/diskann-admin. | |
| # Can also be triggered manually for debugging. | |
| # | |
| # For more details see .github/docs/disk-benchmarks-aa.md | |
| name: Disk Benchmarks (A/A) | |
| on: | |
| schedule: | |
| # Daily at 9 AM UTC | |
| - cron: '0 9 * * *' | |
| workflow_dispatch: # Allow manual trigger for debugging | |
| concurrency: | |
| group: ${{ github.workflow }} | |
| cancel-in-progress: true | |
| env: | |
| RUST_BACKTRACE: 1 | |
| PERF_INPUTS: diskann-benchmark/perf_test_inputs | |
| defaults: | |
| run: | |
| shell: bash | |
| permissions: | |
| contents: read | |
| issues: write # Required for creating failure notification issues | |
| jobs: | |
| # A/A benchmark: run main vs main to detect environment noise. | |
| aa-benchmark: | |
| name: A/A - ${{ matrix.dataset }} | |
| runs-on: [ self-hosted, 1ES.Pool=diskann-github, ubuntu-latest, "JobId=aa-benchmark-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}-${{ strategy.job-index }}" ] | |
| timeout-minutes: 120 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - dataset: wikipedia-100K | |
| config: wikipedia-100K-disk-index.json | |
| archive: wikipedia-100K.tar.gz | |
| - dataset: openai-100K | |
| config: openai-100K-disk-index.json | |
| archive: openai-100K.tar.gz | |
| steps: | |
| # Kept inline because this must run before checkout, but local action.yml | |
| # files are only available after checkout. | |
| - name: Mount high-speed NVMe SSD | |
| shell: bash | |
| run: | | |
| sudo mkdir -p /mnt/nvme | |
| sudo lsblk | |
| sudo mkfs.ext4 /dev/nvme0n1 | |
| sudo mount /dev/nvme0n1 /mnt/nvme | |
| sudo chmod 777 /mnt/nvme | |
| mkdir -p /mnt/nvme/diskann_rust /mnt/nvme/baseline | |
| ln -s /mnt/nvme/diskann_rust diskann_rust | |
| ln -s /mnt/nvme/baseline baseline | |
| - name: Checkout main | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: main | |
| path: diskann_rust | |
| lfs: true | |
| - name: Setup benchmark environment | |
| uses: ./diskann_rust/.github/actions/setup-disk-benchmark | |
| with: | |
| dataset: ${{ matrix.dataset }} | |
| archive: ${{ matrix.archive }} | |
| extract-to: diskann_rust/target/tmp | |
| # A/A: build once, run twice (identical code — only detecting environment noise) | |
| - name: Build benchmark binary | |
| working-directory: diskann_rust | |
| run: cargo build -p diskann-benchmark --features disk-index --release | |
| - name: Run baseline benchmark | |
| working-directory: diskann_rust | |
| run: | | |
| cargo run -p diskann-benchmark --features disk-index --release -- \ | |
| run --input-file ${{ env.PERF_INPUTS }}/${{ matrix.config }} \ | |
| --output-file target/tmp/${{ matrix.dataset }}_baseline.json | |
| - name: Run target benchmark | |
| working-directory: diskann_rust | |
| run: | | |
| cargo run -p diskann-benchmark --features disk-index --release -- \ | |
| run --input-file ${{ env.PERF_INPUTS }}/${{ matrix.config }} \ | |
| --output-file target/tmp/${{ matrix.dataset }}_target.json | |
| - name: Validate benchmark results | |
| working-directory: diskann_rust | |
| run: | | |
| cargo run -p diskann-benchmark --features disk-index --release -- \ | |
| check run \ | |
| --tolerances ${{ env.PERF_INPUTS }}/disk-index-tolerances.json \ | |
| --input-file ${{ env.PERF_INPUTS }}/${{ matrix.config }} \ | |
| --before target/tmp/${{ matrix.dataset }}_baseline.json \ | |
| --after target/tmp/${{ matrix.dataset }}_target.json | |
| - name: Upload benchmark results | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: aa-results-${{ matrix.dataset }} | |
| path: | | |
| diskann_rust/target/tmp/${{ matrix.dataset }}_target.json | |
| diskann_rust/target/tmp/${{ matrix.dataset }}_baseline.json | |
| retention-days: 30 | |
| # Notify diskann-disk-maintainers on A/A failure — but only when the failure | |
| # rate exceeds 5% over the last 20 runs. Our reliability promise is 95%, so | |
| # 1 failure in 20 runs is expected and should not trigger a notification. | |
| notify-on-failure: | |
| name: Notify on A/A Failure | |
| needs: [aa-benchmark] | |
| runs-on: ubuntu-latest | |
| if: failure() | |
| steps: | |
| - name: Check recent failure rate | |
| id: check-rate | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const { data: runs } = await github.rest.actions.listWorkflowRuns({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| workflow_id: 'disk-benchmarks-aa.yml', | |
| per_page: 20, | |
| status: 'completed', | |
| }); | |
| const total = runs.workflow_runs.length; | |
| const failures = runs.workflow_runs.filter(r => r.conclusion === 'failure').length; | |
| const failureRate = total > 0 ? failures / total : 0; | |
| console.log(`Recent A/A runs: ${total}, failures: ${failures}, rate: ${(failureRate * 100).toFixed(1)}%`); | |
| // Only notify if failure rate exceeds our 5% reliability budget | |
| core.setOutput('should_notify', failureRate > 0.05 ? 'true' : 'false'); | |
| core.setOutput('failure_rate', `${(failureRate * 100).toFixed(1)}%`); | |
| core.setOutput('failures', `${failures}`); | |
| core.setOutput('total', `${total}`); | |
| - name: Create GitHub issue for A/A failure | |
| if: steps.check-rate.outputs.should_notify == 'true' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const date = new Date().toISOString().split('T')[0]; | |
| const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; | |
| await github.rest.issues.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| title: `[Benchmark A/A] Daily stability test failed – ${date}`, | |
| body: [ | |
| `## Daily A/A Benchmark Failure`, | |
| ``, | |
| `The scheduled A/A benchmark run (main vs main) **failed** on ${date}.`, | |
| `This indicates environment noise exceeded the configured thresholds.`, | |
| ``, | |
| `**Run:** ${runUrl}`, | |
| `**Recent failure rate:** ${{ steps.check-rate.outputs.failures }}/${{ steps.check-rate.outputs.total }} (${{ steps.check-rate.outputs.failure_rate }}) — exceeds 5% reliability budget`, | |
| ``, | |
| `Please review the benchmark artifacts and determine if thresholds need tuning`, | |
| `or if there is a runner environment issue.`, | |
| ``, | |
| `/cc @microsoft/diskann-disk-maintainers`, | |
| ].join('\n'), | |
| labels: ['benchmark', 'A/A-failure'], | |
| }); |