Publish benchmark results

.github/workflows/benchmark.yaml

@@ -0,0 +1,197 @@
+name: benchmark
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  # Runs on main to cache the benchmark results
+  # or on pull requests to benchmark the target branch (or retrieve the results from the cache).
+  benchmark-base:
+    name: "Base | ${{ matrix.os }}"
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: "Determine base ref"
+        id: base_ref
+        if: github.event_name == 'pull_request'
+        run: echo "ref=${{github.event.pull_request.base.sha || github.sha}}" >> $GITHUB_OUTPUT
+
+      - name: "Restore benchmark results"
+        id: benchmark_cache
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          name: base-benchmark-results-${{ matrix.os }}
+          path: ./target/criterion
+          commit: ${{ steps.base_ref.outputs.ref }}
+          if_no_artifact_found: ignore
+
+      - name: "Checkout Branch"
+        if: steps.benchmark_cache.outputs.found_artifact != 'true'
+        uses: actions/checkout@v3
+        with:
+          ref: ${{steps.base_ref.outputs.ref}}
+
+      - name: "Install Rust toolchain"
+        if: steps.benchmark_cache.outputs.found_artifact != 'true'
+        run: rustup show
+
+      - uses: Swatinem/rust-cache@v1
+        if: steps.benchmark_cache.outputs.found_artifact != 'true'
+
+      - name: "Build benchmarks"
+        uses: actions-rs/cargo@v1
+        if: steps.benchmark_cache.outputs.found_artifact != 'true'
+        with:
+          command: bench
+          args: -p ruff_benchmark --no-run
+
+      - name: "Run benchmarks"
+        if: steps.benchmark_cache.outputs.found_artifact != 'true'
+        run: cargo benchmark --save-baseline=main
+
+      - name: "Upload benchmark results"
+        uses: actions/upload-artifact@v3
+        with:
+          name: base-benchmark-results-${{ matrix.os }}
+          path: ./target/criterion
+
+      # Cleanup
+      - name: Remove Criterion Artifact
+        uses: JesseTG/rm@v1.0.3
+        with:
+          path: ./target/criterion
+
+
+  benchmark-pr:
+    if: github.event_name == 'pull_request'
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    name: "PR | ${{ matrix.os }}"
+
+    steps:
+      - name: "Checkout Branch"
+        uses: actions/checkout@v3
+
+      - name: "Install Rust toolchain"
+        run: rustup show
+
+      - uses: Swatinem/rust-cache@v1
+
+      - name: "Build benchmarks"
+        uses: actions-rs/cargo@v1
+        with:
+          command: bench
+          args: -p ruff_benchmark --no-run
+
+      - name: "Run benchmarks"
+        run: cargo benchmark --save-baseline=pr
+
+      - name: "Upload benchmark results"
+        uses: actions/upload-artifact@v3
+        with:
+          name: pr-benchmark-results-${{ matrix.os }}
+          path: ./target/criterion
+
+      - name: Remove Criterion Artifact
+        uses: JesseTG/rm@v1.0.3
+        with:
+          path: ./target/criterion
+
+  benchmark-compare:
+    runs-on: ubuntu-latest
+    name: "Compare Benchmarks"
+    if: github.event_name == 'pull_request'
+    needs:
+      - benchmark-base
+      - benchmark-pr
+
+    steps:
+    - name: "Install Rust toolchain"
+      run: rustup show
+
+    - name: "Install critcmp"
+      run: cargo install --debug critcmp # Use debug build: Building takes much longer than the "slowness" of using the debug build.
+
+    - name: "Linux | Download PR benchmark results"
+      uses: actions/download-artifact@v3
+      with:
+        name: pr-benchmark-results-ubuntu-latest
+        path: ./target/criterion
+
+    - name: "Linux | Download base benchmark results"
+      uses: actions/download-artifact@v3
+      with:
+        name: base-benchmark-results-ubuntu-latest
+        path: ./target/criterion
+
+    - name: "Linux | Debug"
+      run: ls -al ./target/criterion
+
+    - name: "Linux | Compare benchmark results"
+      shell: bash
+      run: |
+        echo "## Benchmark Results" >> $GITHUB_STEP_SUMMARY
+        echo "### Linux" >> $GITHUB_STEP_SUMMARY
+        echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+        critcmp main pr >> $GITHUB_STEP_SUMMARY
+        echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+
+    - name: "Linux | Cleanup benchmark results"
+      run: rm -rf ./target/criterion
+
+    - name: "Windows | Download PR benchmark results"
+      uses: actions/download-artifact@v3
+      with:
+        name: pr-benchmark-results-windows-latest
+        path: ./target/criterion
+
+    - name: "Windows | Download base benchmark results"
+      uses: actions/download-artifact@v3
+      with:
+        name: base-benchmark-results-windows-latest
+        path: ./target/criterion
+
+    - name: "Windows | Debug"
+      run: ls -al ./target/criterion
+
+    - name: "Windows | Compare benchmark results"
+      shell: bash
+      run: |
+        echo "### Windows" >> $GITHUB_STEP_SUMMARY
+        echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+        critcmp main pr >> $GITHUB_STEP_SUMMARY
+        echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+
+    - name: Find Comment
+      # Check if the event is not triggered by a fork
+      if: github.event.pull_request.head.repo.full_name == github.repository
+      uses: peter-evans/find-comment@v2
+      id: fc
+      with:
+        issue-number: ${{ github.event.pull_request.number }}
+        comment-author: 'github-actions[bot]'
+        body-includes: Benchmark Results
+
+    - name: Create or update comment
+      # Check if the event is not triggered by a fork
+      if: github.event.pull_request.head.repo.full_name == github.repository
+      uses: peter-evans/create-or-update-comment@v2
+      with:
+        issue-number: ${{ github.event.pull_request.number }}
+        edit-mode: replace
+        comment-id: ${{ steps.fc.outputs.comment-id }}
+        body: $GITHUB_STEP_SUMMARY