Linux benchmark on CI (#1809)

.github/workflows/linux-ci.yml

@@ -93,16 +93,10 @@ jobs:
           CI: 1
         run: |
           cmake --version
-          cmake -B build -GNinja -DCMAKE_BUILD_TYPE=Debug -DMLN_WITH_CLANG_TIDY=ON -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DMLN_WITH_COVERAGE=ON ${{ env.renderer_flag_cmake }}
-          cmake --build build --target mbgl-core mbgl-test-runner mbgl-render-test-runner mbgl-expression-test
+          cmake -B build -GNinja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMLN_WITH_CLANG_TIDY=ON -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DMLN_WITH_COVERAGE=ON ${{ env.renderer_flag_cmake }}
+          cmake --build build --target mbgl-core mbgl-test-runner mbgl-render-test-runner mbgl-expression-test mbgl-render mbgl-benchmark-runner
 
-      # mbgl-render (used for size test)
-
-      - name: Build mbgl-render
-        if: matrix.renderer == 'drawable'
-        run: |
-          cmake -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -B build-size -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DMLN_DRAWABLE_RENDERER=ON
-          cmake --build build-size --target mbgl-render
+      # mbgl-render (used for size test) & mbgl-benchmark-runner
 
       - name: Upload mbgl-render as artifact
         if: matrix.renderer == 'drawable' && github.event_name == 'pull_request' 
@@ -110,7 +104,15 @@ jobs:
         with:
           name: mbgl-render
           path: |
-            build-size/bin/mbgl-render
+            build/bin/mbgl-render
+  
+      - name: Upload mbgl-benchmark-runner as artifact
+        if: matrix.renderer == 'drawable' && github.event_name == 'pull_request' 
+        uses: actions/upload-artifact@v3
+        with:
+          name: mbgl-benchmark-runner
+          path: |
+            build/mbgl-benchmark-runner
 
       - name: Configure AWS Credentials
         if: matrix.renderer == 'drawable' && github.ref == 'refs/heads/main' && vars.OIDC_AWS_ROLE_TO_ASSUME 
@@ -120,9 +122,11 @@ jobs:
           role-to-assume: ${{ vars.OIDC_AWS_ROLE_TO_ASSUME }}
           role-session-name: ${{ github.run_id }}
 
-      - name: Upload mbgl-render to S3
+      - name: Upload mbgl-render & mbgl-benchmark-runner to S3
         if: matrix.renderer == 'drawable' && github.ref == 'refs/heads/main' && vars.OIDC_AWS_ROLE_TO_ASSUME 
-        run: aws s3 cp build-size/bin/mbgl-render s3://maplibre-native/mbgl-render-main
+        run: | 
+          aws s3 cp build/bin/mbgl-render s3://maplibre-native/mbgl-render-main
+          aws s3 cp build/mbgl-benchmark-runner s3://maplibre-native/mbgl-benchmark-runner-main
 
       # CodeQL
 
@@ -225,4 +229,4 @@ jobs:
     steps:
       - name: Mark result as failed
         if: needs.linux-build-and-test.result != 'success' || needs.linux-coverage.result != 'success'
-        run: exit 1
+        run: exit 1

.github/workflows/pr-tests.yml

@@ -1,6 +1,5 @@
-
-
-name: pr-bloaty
+# Runs Bloaty (size test) and Google Benchmark and reports results on PR
+name: pr-tests
 
 on:
   workflow_run:
@@ -88,9 +87,8 @@ jobs:
             awk 'NR <= 2; END { print }' tlus_diff_main.txt
             echo '```'
             echo "Full report: $download_url/$report_path"
+            echo "---"
           } >> message.md
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Prepare Bloaty message (legacy)
         run: |
@@ -103,8 +101,6 @@ jobs:
             echo '```'
             echo "Full report: $download_url/$report_path"
           } >> message.md
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Leave a comment with Bloaty results
         uses: marocchino/sticky-pull-request-comment@v2
@@ -112,3 +108,55 @@ jobs:
           number: ${{ steps.get-pr-number.outputs.pr-number }}
           header: bloaty
           path: message.md
+
+  pr-benchmark:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ./.github/actions/get-pr-number
+        id: get-pr-number
+
+      - uses: ./.github/actions/download-workflow-run-artifact
+        with:
+          artifact-name: mbgl-benchmark-runner
+          expect-files: "mbgl-benchmark-runner"
+
+      - name: Download mbgl-benchmark-runner-main
+        run: wget -O mbgl-benchmark-runner-main "${download_url}/mbgl-benchmark-runner-main"
+
+      # needed for compare.py
+      - name: Install scipy
+        run: pip3 install scipy
+
+      - name: Run Benchmarks
+        # excluding the API tests because they hang https://github.com/maplibre/maplibre-native/issues/1808
+        run: python3 vendor/benchmark/tools/compare.py benchmarks mbgl-benchmark-runner-main mbgl-benchmark-runner --benchmark_filter='^[^A][^P][^I].*' > benchmark_out.txt
+
+      - name: Configure AWS Credentials
+        if: github.ref == 'refs/heads/main' && vars.OIDC_AWS_ROLE_TO_ASSUME 
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: us-west-2
+          role-to-assume: ${{ vars.OIDC_AWS_ROLE_TO_ASSUME }}
+          role-session-name: ${{ github.run_id }}
+
+      - name: Prepare Benchmark message
+        run: |
+          sed -n '/Comparing/,$p' benchmark_out.txt | tail -n +2 > benchmark_diff.txt
+          report_path=benchmark-results/pr-${{ steps.get-pr-number.outputs.pr-number }}-compared-to-main.txt
+          aws s3 cp benchmark_diff.txt s3://maplibre-native/"$report_path"
+          {
+            echo "Benchmark Results ⚡"
+            echo '```'
+            awk 'NR <= 2; END { print }' benchmark_diff.txt
+            echo '```'
+            echo "Full report: $download_url/$report_path"
+          } >> message.md
+
+      - name: Leave a comment with Benchmark results
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          number: ${{ steps.get-pr-number.outputs.pr-number }}
+          header: benchmark
+          path: message.md