name: Performance Tests
on:
pull_request:
push:
branches:
- main
workflow_dispatch:
inputs:
compare_upstream:
type: boolean
default: true
description: Compare against upstream quinn
publish_metrics:
type: boolean
default: false
description: Publish metrics to perf.iroh.computer
concurrency:
group: perf-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
RUST_BACKTRACE: 1
RUSTFLAGS: -Dwarnings
SCCACHE_GHA_ENABLED: "true"
RUSTC_WRAPPER: "sccache"
jobs:
build:
name: Build ${{ matrix.impl }} perf
runs-on: [self-hosted, linux, X64]
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
include:
- impl: iroh-quinn
artifact: quinn-perf-iroh
- impl: upstream-quinn
artifact: quinn-perf-upstream
steps:
- uses: actions/checkout@v6
if: matrix.impl == 'iroh-quinn'
- uses: dtolnay/rust-toolchain@stable
- uses: mozilla-actions/sccache-action@v0.0.9
- name: Get latest quinn release tag
if: matrix.impl == 'upstream-quinn'
id: quinn-release
run: |
LATEST_TAG=$(curl -s https://api.github.com/repos/quinn-rs/quinn/releases/latest | jq -r '.tag_name')
echo "tag=$LATEST_TAG" >> $GITHUB_OUTPUT
echo "Using upstream quinn release: $LATEST_TAG"
- name: Clone upstream quinn
if: matrix.impl == 'upstream-quinn'
run: |
rm -rf /tmp/upstream-quinn
git clone --depth 1 --branch ${{ steps.quinn-release.outputs.tag }} \
https://github.com/quinn-rs/quinn.git /tmp/upstream-quinn
- name: Build perf binary
run: cargo build --release -p perf --features json-output
working-directory: ${{ matrix.impl == 'upstream-quinn' && '/tmp/upstream-quinn' || '.' }}
- name: Upload binary
uses: actions/upload-artifact@v5
with:
name: ${{ matrix.artifact }}
path: ${{ matrix.impl == 'upstream-quinn' && '/tmp/upstream-quinn/target/release/quinn-perf' || 'target/release/quinn-perf' }}
retention-days: 1
raw-perf:
name: Raw perf ${{ matrix.impl }}
needs: build
runs-on: [self-hosted, linux, X64]
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
impl: [iroh-quinn, upstream-quinn]
include:
- impl: iroh-quinn
artifact: quinn-perf-iroh
- impl: upstream-quinn
artifact: quinn-perf-upstream
steps:
- uses: actions/checkout@v6
- uses: actions/download-artifact@v5
with:
name: ${{ matrix.artifact }}
path: ./bin
- name: Run benchmarks
run: |
chmod +x ./bin/quinn-perf
./.github/scripts/run_perf.sh ./bin/quinn-perf results/${{ matrix.impl }}
- uses: actions/upload-artifact@v5
with:
name: raw-results-${{ matrix.impl }}
path: results/
retention-days: 3
netsim-perf:
name: Netsim perf ${{ matrix.impl }}
needs: build
runs-on: [self-hosted, linux, X64]
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
impl: [iroh-quinn, upstream-quinn]
include:
- impl: iroh-quinn
artifact: quinn-perf-iroh
- impl: upstream-quinn
artifact: quinn-perf-upstream
steps:
- uses: actions/checkout@v6
- uses: actions/download-artifact@v5
with:
name: ${{ matrix.artifact }}
path: ./bin
- name: Fetch netsim
run: |
rm -rf /tmp/chuck
git clone --depth 1 https://github.com/n0-computer/chuck.git /tmp/chuck
- name: Install netsim deps
run: |
cd /tmp/chuck/netsim
sudo apt-get update
./setup.sh || true
./cleanup.sh || true
- name: Setup netsim bins
run: |
chmod +x ./bin/quinn-perf
mkdir -p /tmp/chuck/netsim/bins
cp ./bin/quinn-perf /tmp/chuck/netsim/bins/
- name: Run netsim perf test
id: run_netsim
continue-on-error: true
run: |
./.github/scripts/run_netsim_perf.sh ${{ matrix.impl }} /tmp/chuck/netsim
- name: Collect results
if: always()
run: |
mkdir -p results/${{ matrix.impl }}
cp /tmp/chuck/netsim/report/*.json results/${{ matrix.impl }}/ 2>/dev/null || true
cp /tmp/chuck/netsim/logs/*.log results/${{ matrix.impl }}/ 2>/dev/null || true
- uses: actions/upload-artifact@v5
if: always()
with:
name: netsim-results-${{ matrix.impl }}
path: results/
retention-days: 3
- name: Cleanup netsim
if: always()
run: |
cd /tmp/chuck/netsim
./cleanup.sh || true
compare:
name: Compare results
needs: [raw-perf, netsim-perf]
if: always()
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v6
- uses: actions/download-artifact@v5
with:
pattern: raw-results-*
merge-multiple: true
path: raw-results/
- uses: actions/download-artifact@v5
with:
pattern: netsim-results-*
merge-multiple: true
path: netsim-results/
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: '3.11'
- name: Generate comparison report
id: compare
run: |
python .github/scripts/compare_results.py raw-results/ netsim-results/ > comparison.md
echo "COMPARISON<<EOF" >> $GITHUB_OUTPUT
cat comparison.md >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
- name: Convert to Metro format
run: |
python .github/scripts/perf_to_metro.py raw-results/ netsim-results/ \
--commit ${{ github.sha }} > metro.json
- name: Show results
run: |
echo "=== Comparison Report ==="
cat comparison.md
echo ""
echo "=== Metro Metrics ==="
cat metro.json
- name: Upload comparison report
uses: actions/upload-artifact@v5
with:
name: perf-comparison-report
path: |
comparison.md
metro.json
retention-days: 30
- name: Publish metrics to Metro
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
run: |
if -n "${{ secrets.METRO_TOKEN }}" && -n "${{ secrets.METRO_ENDPOINT }}" ; then
curl -X POST \
-H "Content-Type: application/json" \
-H "Authorization: Bearer ${{ secrets.METRO_TOKEN }}" \
--data @metro.json \
"${{ secrets.METRO_ENDPOINT }}"
else
echo "Metro secrets not configured, skipping publish"
fi
- name: Find existing comment
if: github.event_name == 'pull_request'
uses: peter-evans/find-comment@v3
id: find-comment
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: 'github-actions[bot]'
body-includes: '<!-- perf-report -->'
- name: Create new PR comment
if: github.event_name == 'pull_request' && steps.find-comment.outputs.comment-id == ''
uses: peter-evans/create-or-update-comment@v5
with:
issue-number: ${{ github.event.pull_request.number }}
body: |
<!-- perf-report -->
## Performance Comparison Report
<details>
<summary><code>${{ github.event.pull_request.head.sha }} - - name: Update existing PR comment
if: github.event_name == 'pull_request' && steps.find-comment.outputs.comment-id != ''
uses: peter-evans/create-or-update-comment@v5
with:
comment-id: ${{ steps.find-comment.outputs.comment-id }}
edit-mode: append
body: |
---
<details>
<summary><code>${{ github.event.pull_request.head.sha }} -
Graph