From c2d91477b1955b7ffda335de645f1c74f136463b Mon Sep 17 00:00:00 2001 From: Oliver Wipfli Date: Sat, 13 Aug 2022 12:42:10 +0200 Subject: [PATCH] Run render tests in CI (#357) * Add render test workflow * Sudo * Add debug output * Remove path * Use xvfb * Remove unneeded code * Upload render query test artifact * Always upload artifact * Add failing tests to ignores file * Rename workflow * Fix file extension * Change xvfb options * Ignore x display errors * Run render tests on macOS * Upload render test artifacts * Run linux render tests * Revert runner changes * Remove linux render test * Remove ignores * Add macos ignore * Add documentation * Update DEVELOPING.md Co-authored-by: Tadej Novak --- .github/workflows/node-ci.yml | 11 +++++ DEVELOPING.md | 58 +++++++++++++++++++++++- metrics/ignores/platform-linux.json | 1 + metrics/ignores/platform-macos.json | 1 + metrics/macos-xcode11-release-style.json | 3 +- 5 files changed, 72 insertions(+), 2 deletions(-) diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml index 3479dfece..e5f128119 100644 --- a/.github/workflows/node-ci.yml +++ b/.github/workflows/node-ci.yml @@ -163,6 +163,17 @@ jobs: run: | cmake --build build -j $(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null) + - name: Run render tests on macOS + if: runner.os == 'macOS' + run: ./build/mbgl-render-test-runner --manifestPath metrics/macos-xcode11-release-style.json + + - name: Upload render test artifacts + uses: actions/upload-artifact@v3 + if: always() + with: + name: render-query-test-results + path: metrics/macos-xcode11-release-style.html + - name: Test if: runner.os == 'Linux' run: xvfb-run --auto-servernum npm test diff --git a/DEVELOPING.md b/DEVELOPING.md index 70f43d697..abbdbfcfd 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -6,7 +6,63 @@ This is been rewritten to include: - Acceptance criteria for code contributions (style, static asserts) - How to run the unit tests. - How to run the benchmarks. -- How to run the render tests. - How to rebaseline baselines metrics. - How to use GL Native as a 3rd party library in your project. +## Render Tests + +To check that the output of the rendering is correct, we compare actual rendered PNGs for simple styles with expected PNGs. The content of the tests is stored in the MapLibre GL JS submodule which means that GL JS and Native are in fact quasi pixel-identical in their rendering. + +The directory sturcture of the render tests looks like: + +``` +maplibre-gl-js/ + test/ + integration/ + render-tests/ + / + / + expected.png + style.json +``` + +After the render test run, the folder will also contain an `actual.png` file and a `diff.png` which is the difference between the expected and the actual image. There is a pixel difference threshold value which is used to decide if a render test passed or failed. + +### Building the render test runner + +Run the following on linux to build the render test runner: + +``` +git submodule update --init --recursive --depth 1 + +sudo apt update +sudo apt install ccache cmake ninja-build pkg-config xvfb libcurl4-openssl-dev libglfw3-dev libuv1-dev g++-10 libc++-9-dev libc++abi-9-dev libpng-dev libgl1-mesa-dev libgl1-mesa-dri libjpeg-turbo8 libicu66 libjpeg-dev + +cmake . -B build -G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=gcc-10 -DCMAKE_CXX_COMPILER=g++-10 +cmake --build build -j $(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null) +``` + + +### Running the render tests + +On linux, run all render tests with: + +``` +./build/mbgl-render-test-runner --manifestPath metrics/linux-clang8-release-style.json +``` + +Or a single test with: + +``` +./build/mbgl-render-test-runner --manifestPath metrics/linux-clang8-release-style.json --filter "render-tests/fill-visibility/visible" +``` + +On macOS, run all render tests with: + +``` +./build/mbgl-render-test-runner --manifestPath metrics/macos-xcode11-release-style.json +``` + +### Inspecting render test results + +The render test results are summarized in a HTML website located next to the manifest file. For example, running `metrics/linux-clang8-release-style.json` produces a summary at `metrics/linux-clang8-release-style.html`. diff --git a/metrics/ignores/platform-linux.json b/metrics/ignores/platform-linux.json index 9d3f0d491..de5bf88d9 100644 --- a/metrics/ignores/platform-linux.json +++ b/metrics/ignores/platform-linux.json @@ -1,4 +1,5 @@ { + "render-tests/icon-text-fit/enlargen-both": "Not working for unknown reason: https://github.com/maplibre/maplibre-gl-native/issues/355", "render-tests/fill-opacity/zoom-and-property-function-pattern": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/15322", "render-tests/fill-pattern/literal": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/14423", "render-tests/fill-pattern/opacity": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/14870", diff --git a/metrics/ignores/platform-macos.json b/metrics/ignores/platform-macos.json index bc5e67820..8c1bd921e 100644 --- a/metrics/ignores/platform-macos.json +++ b/metrics/ignores/platform-macos.json @@ -1,4 +1,5 @@ { + "render-tests/icon-image/icon-sdf-non-sdf-one-layer": "Failing on GitHub runner for unknown reason https://github.com/maplibre/maplibre-gl-native/issues/377", "render-tests/regressions/mapbox-gl-js#5642": "Failing with mbgl-render-test", "render-tests/regressions/mapbox-gl-js#7066": "Failing with mbgl-render-test" } diff --git a/metrics/macos-xcode11-release-style.json b/metrics/macos-xcode11-release-style.json index ffeb381ce..bfde0ec04 100644 --- a/metrics/macos-xcode11-release-style.json +++ b/metrics/macos-xcode11-release-style.json @@ -6,7 +6,8 @@ ], "ignore_paths": [ "ignores/platform-all.json", - "ignores/platform-macos.json" + "ignores/platform-macos.json", + "ignores/platform-linux.json" ], "metric_path": "macos-xcode11-release", "probes": [