From cd0db4a1539700670e97884c1abcfc24feeab5e9 Mon Sep 17 00:00:00 2001 From: Bart Louwers Date: Thu, 26 Oct 2023 22:24:27 +0200 Subject: [PATCH] Linux benchmark on CI (#1809) --- .github/workflows/linux-ci.yml | 30 +++++---- .../workflows/{pr-bloaty.yml => pr-tests.yml} | 62 ++++++++++++++++--- vendor/benchmark | 2 +- vendor/benchmark.cmake | 3 +- 4 files changed, 75 insertions(+), 22 deletions(-) rename .github/workflows/{pr-bloaty.yml => pr-tests.yml} (63%) diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index 58e0af7cf..3a8ccd3bf 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -93,16 +93,10 @@ jobs: CI: 1 run: | cmake --version - cmake -B build -GNinja -DCMAKE_BUILD_TYPE=Debug -DMLN_WITH_CLANG_TIDY=ON -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DMLN_WITH_COVERAGE=ON ${{ env.renderer_flag_cmake }} - cmake --build build --target mbgl-core mbgl-test-runner mbgl-render-test-runner mbgl-expression-test + cmake -B build -GNinja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMLN_WITH_CLANG_TIDY=ON -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DMLN_WITH_COVERAGE=ON ${{ env.renderer_flag_cmake }} + cmake --build build --target mbgl-core mbgl-test-runner mbgl-render-test-runner mbgl-expression-test mbgl-render mbgl-benchmark-runner - # mbgl-render (used for size test) - - - name: Build mbgl-render - if: matrix.renderer == 'drawable' - run: | - cmake -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -B build-size -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DMLN_DRAWABLE_RENDERER=ON - cmake --build build-size --target mbgl-render + # mbgl-render (used for size test) & mbgl-benchmark-runner - name: Upload mbgl-render as artifact if: matrix.renderer == 'drawable' && github.event_name == 'pull_request' @@ -110,7 +104,15 @@ jobs: with: name: mbgl-render path: | - build-size/bin/mbgl-render + build/bin/mbgl-render + + - name: Upload mbgl-benchmark-runner as artifact + if: matrix.renderer == 'drawable' && github.event_name == 'pull_request' + uses: actions/upload-artifact@v3 + with: + name: mbgl-benchmark-runner + path: | + build/mbgl-benchmark-runner - name: Configure AWS Credentials if: matrix.renderer == 'drawable' && github.ref == 'refs/heads/main' && vars.OIDC_AWS_ROLE_TO_ASSUME @@ -120,9 +122,11 @@ jobs: role-to-assume: ${{ vars.OIDC_AWS_ROLE_TO_ASSUME }} role-session-name: ${{ github.run_id }} - - name: Upload mbgl-render to S3 + - name: Upload mbgl-render & mbgl-benchmark-runner to S3 if: matrix.renderer == 'drawable' && github.ref == 'refs/heads/main' && vars.OIDC_AWS_ROLE_TO_ASSUME - run: aws s3 cp build-size/bin/mbgl-render s3://maplibre-native/mbgl-render-main + run: | + aws s3 cp build/bin/mbgl-render s3://maplibre-native/mbgl-render-main + aws s3 cp build/mbgl-benchmark-runner s3://maplibre-native/mbgl-benchmark-runner-main # CodeQL @@ -225,4 +229,4 @@ jobs: steps: - name: Mark result as failed if: needs.linux-build-and-test.result != 'success' || needs.linux-coverage.result != 'success' - run: exit 1 \ No newline at end of file + run: exit 1 diff --git a/.github/workflows/pr-bloaty.yml b/.github/workflows/pr-tests.yml similarity index 63% rename from .github/workflows/pr-bloaty.yml rename to .github/workflows/pr-tests.yml index dfe20d89d..9850c3ebd 100644 --- a/.github/workflows/pr-bloaty.yml +++ b/.github/workflows/pr-tests.yml @@ -1,6 +1,5 @@ - - -name: pr-bloaty +# Runs Bloaty (size test) and Google Benchmark and reports results on PR +name: pr-tests on: workflow_run: @@ -88,9 +87,8 @@ jobs: awk 'NR <= 2; END { print }' tlus_diff_main.txt echo '```' echo "Full report: $download_url/$report_path" + echo "---" } >> message.md - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Prepare Bloaty message (legacy) run: | @@ -103,8 +101,6 @@ jobs: echo '```' echo "Full report: $download_url/$report_path" } >> message.md - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Leave a comment with Bloaty results uses: marocchino/sticky-pull-request-comment@v2 @@ -112,3 +108,55 @@ jobs: number: ${{ steps.get-pr-number.outputs.pr-number }} header: bloaty path: message.md + + pr-benchmark: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/get-pr-number + id: get-pr-number + + - uses: ./.github/actions/download-workflow-run-artifact + with: + artifact-name: mbgl-benchmark-runner + expect-files: "mbgl-benchmark-runner" + + - name: Download mbgl-benchmark-runner-main + run: wget -O mbgl-benchmark-runner-main "${download_url}/mbgl-benchmark-runner-main" + + # needed for compare.py + - name: Install scipy + run: pip3 install scipy + + - name: Run Benchmarks + # excluding the API tests because they hang https://github.com/maplibre/maplibre-native/issues/1808 + run: python3 vendor/benchmark/tools/compare.py benchmarks mbgl-benchmark-runner-main mbgl-benchmark-runner --benchmark_filter='^[^A][^P][^I].*' > benchmark_out.txt + + - name: Configure AWS Credentials + if: github.ref == 'refs/heads/main' && vars.OIDC_AWS_ROLE_TO_ASSUME + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-west-2 + role-to-assume: ${{ vars.OIDC_AWS_ROLE_TO_ASSUME }} + role-session-name: ${{ github.run_id }} + + - name: Prepare Benchmark message + run: | + sed -n '/Comparing/,$p' benchmark_out.txt | tail -n +2 > benchmark_diff.txt + report_path=benchmark-results/pr-${{ steps.get-pr-number.outputs.pr-number }}-compared-to-main.txt + aws s3 cp benchmark_diff.txt s3://maplibre-native/"$report_path" + { + echo "Benchmark Results ⚡" + echo '```' + awk 'NR <= 2; END { print }' benchmark_diff.txt + echo '```' + echo "Full report: $download_url/$report_path" + } >> message.md + + - name: Leave a comment with Benchmark results + uses: marocchino/sticky-pull-request-comment@v2 + with: + number: ${{ steps.get-pr-number.outputs.pr-number }} + header: benchmark + path: message.md \ No newline at end of file diff --git a/vendor/benchmark b/vendor/benchmark index 090faecb4..344117638 160000 --- a/vendor/benchmark +++ b/vendor/benchmark @@ -1 +1 @@ -Subproject commit 090faecb454fbd6e6e17a75ef8146acb037118d4 +Subproject commit 344117638c8ff7e239044fd0fa7085839fc03021 diff --git a/vendor/benchmark.cmake b/vendor/benchmark.cmake index d8cf998dc..b3927868f 100644 --- a/vendor/benchmark.cmake +++ b/vendor/benchmark.cmake @@ -9,6 +9,7 @@ add_library( ${CMAKE_CURRENT_LIST_DIR}/benchmark/src/benchmark_name.cc ${CMAKE_CURRENT_LIST_DIR}/benchmark/src/benchmark_register.cc ${CMAKE_CURRENT_LIST_DIR}/benchmark/src/benchmark_runner.cc + ${CMAKE_CURRENT_LIST_DIR}/benchmark/src/check.cc ${CMAKE_CURRENT_LIST_DIR}/benchmark/src/colorprint.cc ${CMAKE_CURRENT_LIST_DIR}/benchmark/src/commandlineflags.cc ${CMAKE_CURRENT_LIST_DIR}/benchmark/src/complexity.cc @@ -16,8 +17,8 @@ add_library( ${CMAKE_CURRENT_LIST_DIR}/benchmark/src/counter.cc ${CMAKE_CURRENT_LIST_DIR}/benchmark/src/csv_reporter.cc ${CMAKE_CURRENT_LIST_DIR}/benchmark/src/json_reporter.cc + ${CMAKE_CURRENT_LIST_DIR}/benchmark/src/perf_counters.cc ${CMAKE_CURRENT_LIST_DIR}/benchmark/src/reporter.cc - ${CMAKE_CURRENT_LIST_DIR}/benchmark/src/sleep.cc ${CMAKE_CURRENT_LIST_DIR}/benchmark/src/statistics.cc ${CMAKE_CURRENT_LIST_DIR}/benchmark/src/string_util.cc ${CMAKE_CURRENT_LIST_DIR}/benchmark/src/sysinfo.cc