Cache main benchmark binary, merge with ecosystem-results comment

.github/workflows/benchmark.yaml

@@ -1,6 +1,8 @@
-name: benchmark
+name: Benchmark
 
 on:
+  push:
+    branches: [ main ]
   pull_request:
   workflow_dispatch:
 
@@ -9,8 +11,38 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  build-baseline-binary:
+    if: github.event_name != 'pull_request'
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, windows-latest ]
+    runs-on: ${{ matrix.os }}
+    name: Build Benchmark Binary | ${{ matrix.os }}
+
+    steps:
+      - name: "Checkout Branch"
+        uses: actions/checkout@v3
+
+      - name: "Install Rust toolchain"
+        run: rustup show
+
+      - uses: Swatinem/rust-cache@v1
+
+      - name: "Build benchmarks"
+        uses: actions-rs/cargo@v1
+        with:
+          command: bench
+          args: -p ruff_benchmark --no-run
+
+      - name: "Save Binary"
+        uses: actions/upload-artifact@v3
+        with:
+          name: ruff-benchmark-${{ matrix.os }}
+          path: ./target/release/ruff_benchmark
+
   run-benchmark:
-    name: "Base | ${{ matrix.os }}"
+    if: github.event_name == 'pull_request'
+    name: "Run | ${{ matrix.os }}"
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
@@ -36,16 +68,33 @@ jobs:
       - name: "PR - Run benchmarks"
         run: cargo benchmark --save-baseline=pr
 
+      - name: "Main - Download pre-built binary"
+        uses: dawidd6/action-download-artifact@v2
+        id: download_base_bin
+        with:
+          name: ruff-benchmark-${{ matrix.os }}
+          commit: ${{ github.event.pull_request.base.sha }}
+          path: ./target/release/ruff_benchmark
+          if_no_artifact_found: ignore
+          search_artifacts: true # Necessary because of build matrix (different run for each os)
+
+      - name: "Main - Set executable flag"
+        if: steps.download_base_bin.outputs.found_artifact == 'true'
+        run: chmod +x ./target/release/ruff_benchmark
+
       - name: "Main - Checkout Branch"
+        if: steps.download_base_bin.outputs.found_artifact != 'true'
         uses: actions/checkout@v3
         with:
           clean: false
           ref: main
 
       - name: "Main - Install Rust toolchain"
+        if: steps.download_base_bin.outputs.found_artifact != 'true'
         run: rustup show
 
       - name: "Main - Build benchmarks"
+        if: steps.download_base_bin.outputs.found_artifact != 'true'
         uses: actions-rs/cargo@v1
         with:
           command: bench
@@ -67,8 +116,9 @@ jobs:
           path: ./target/criterion
 
   benchmark-compare:
+    if: github.event_name == 'pull_request'
     runs-on: ubuntu-latest
-    name: "Compare Benchmarks"
+    name: Compare
     needs:
       - run-benchmark
 
@@ -77,7 +127,8 @@ jobs:
       run: rustup show
 
     - name: "Install critcmp"
-      run: cargo install --debug critcmp # Use debug build: Building takes much longer than the "slowness" of using the debug build.
+      # Use debug build: Building takes much longer than the "slowness" of using the debug build.
+      run: cargo install --debug critcmp
 
     - name: "Linux | Download PR benchmark results"
       uses: actions/download-artifact@v3
@@ -88,8 +139,8 @@ jobs:
     - name: "Linux | Compare benchmark results"
       shell: bash
       run: |
-        echo "## Benchmark Results" >> summary.md
-        echo "### Linux" >> summary.md
+        echo "### Benchmark Results" >> summary.md
+        echo "#### Linux" >> summary.md
         echo "\`\`\`" >> summary.md
         critcmp main pr >> summary.md
         echo "\`\`\`" >> summary.md
@@ -107,29 +158,24 @@ jobs:
     - name: "Windows | Compare benchmark results"
       shell: bash
       run: |
-        echo "### Windows" >> summary.md
+        echo "#### Windows" >> summary.md
         echo "\`\`\`" >> summary.md
         critcmp main pr >> summary.md
         echo "\`\`\`" >> summary.md
         echo "" >> summary.md
+
+        echo ${{ github.event.pull_request.number }} > pr-number
+
         cat summary.md > $GITHUB_STEP_SUMMARY
 
-    - name: Find Comment
-      # Check if the event is not triggered by a fork
-      if: github.event.pull_request.head.repo.full_name == github.repository
-      uses: peter-evans/find-comment@v2
-      id: fc
+    - uses: actions/upload-artifact@v3
+      name: Upload PR Number
       with:
-        issue-number: ${{ github.event.pull_request.number }}
-        comment-author: 'github-actions[bot]'
-        body-includes: Benchmark Results
-
-    - name: Create or update comment
-      # Check if the event is not triggered by a fork
-      if: github.event.pull_request.head.repo.full_name == github.repository
-      uses: peter-evans/create-or-update-comment@v2
+        name: pr-number
+        path: pr-number
+
+    - uses: actions/upload-artifact@v3
+      name: Upload Summary
       with:
-        issue-number: ${{ github.event.pull_request.number }}
-        edit-mode: replace
-        comment-id: ${{ steps.fc.outputs.comment-id }}
-        body-file: summary.md
+        name: summary
+        path: summary.md

.github/workflows/ci.yaml

@@ -80,6 +80,7 @@ jobs:
           # Setting RUSTDOCFLAGS because `cargo doc --check` isn't yet implemented (https://github.com/rust-lang/cargo/issues/10025).
           RUSTDOCFLAGS: "-D warnings"
       - uses: actions/upload-artifact@v3
+        if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
           name: ruff
           path: target/debug/ruff
@@ -139,27 +140,39 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.11"
+
       - uses: actions/download-artifact@v3
+        name: Download Ruff binary
         id: ruff-target
         with:
           name: ruff
           path: target/debug
+
       - uses: dawidd6/action-download-artifact@v2
+        name: Download base results
         with:
           name: ruff
           branch: ${{ github.event.pull_request.base.ref }}
           check_artifacts: true
+
       - name: Run ecosystem check
         run: |
           # Make executable, since artifact download doesn't preserve this
           chmod +x ruff ${{ steps.ruff-target.outputs.download-path }}/ruff
 
           scripts/check_ecosystem.py ruff ${{ steps.ruff-target.outputs.download-path }}/ruff | tee ecosystem-result
+          cat ecosystem-result > $GITHUB_STEP_SUMMARY
 
           echo ${{ github.event.number }} > pr-number
+
+      - uses: actions/upload-artifact@v3
+        name: Upload PR Number
+        with:
+          name: pr-number
+          path: pr-number
+
       - uses: actions/upload-artifact@v3
+        name: Upload Results
         with:
           name: ecosystem-result
-          path: |
-            ecosystem-result
-            pr-number
+          path: ecosystem-result

.github/workflows/pr-comment.yaml

@@ -2,7 +2,7 @@ name: PR Check Comment
 
 on:
   workflow_run:
-    workflows: [CI]
+    workflows: [CI, Benchmark]
     types: [completed]
   workflow_dispatch:
     inputs:
@@ -17,22 +17,66 @@ jobs:
   comment:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
       - uses: dawidd6/action-download-artifact@v2
-        id: download-result
+        name: Download PR Number
+        with:
+          name: pr-number
+          run_id: ${{ github.event.workflow_run.id ||  github.event.inputs.workflow_run_id }}
+          if_no_artifact_found: ignore
+
+      - name: Extract PR Number
+        id: pr-number
+        run: |
+          if [[ -f pr-number ]]
+          then
+            echo "pr-number=$(<pr-number)" >> $GITHUB_OUTPUT
+          fi
+
+      - uses: dawidd6/action-download-artifact@v2
+        name: "Download Ecosystem Result"
+        id: download-ecosystem-result
+        if: steps.pr-number.outputs.pr-number
         with:
           name: ecosystem-result
           workflow: ci.yaml
-          run_id: ${{ github.event.workflow_run.id || github.event.inputs.workflow_run_id }}
+          pr: ${{ steps.pr-number.outputs.pr-number }}
+          path: pr/ecosystem
           if_no_artifact_found: ignore
-      - if: steps.download-result.outputs.found_artifact
-        id: result
+
+      - uses: dawidd6/action-download-artifact@v2
+        name: "Download Benchmark Result"
+        id: download-benchmark-result
+        if: steps.pr-number.outputs.pr-number
+        with:
+          name: summary
+          workflow: benchmark.yaml
+          pr: ${{ steps.pr-number.outputs.pr-number }}
+          path: pr/benchmark
+          if_no_artifact_found: ignore
+
+      - name: Generate Comment
+        id: generate-comment
+        if: steps.download-ecosystem-result.outputs.found_artifact == 'true' || steps.download-benchmark-result.outputs.found_artifact == 'true'
         run: |
-          echo "pr-number=$(<pr-number)" >> $GITHUB_OUTPUT
-      - name: Create comment
-        if: steps.download-result.outputs.found_artifact
+          echo 'comment<<EOF' >> $GITHUB_OUTPUT
+          echo '## PR Check Results' >> $GITHUB_OUTPUT
+
+          if [[ -f pr/ecosystem/ecosystem-result ]]
+          then
+            cat pr/ecosystem/ecosystem-result >> $GITHUB_OUTPUT
+          fi
+
+          if [[ -f pr/benchmark/summary.md ]]
+          then
+            cat pr/benchmark/summary.md >> $GITHUB_OUTPUT
+          fi
+
+          echo 'EOF' >> $GITHUB_OUTPUT
+
+      - name: Create or update comment
+        if: steps.generate-comment.outputs.comment
         uses: thollander/actions-comment-pull-request@v2
         with:
-          pr_number: ${{ steps.result.outputs.pr-number }}
-          filePath: ecosystem-result
-          comment_tag: ecosystem-results
+          pr_number: ${{ steps.pr-number.outputs.pr-number }}
+          message: ${{ steps.generate-comment.outputs.comment }}
+          comment_tag: PR Check Results