[build] Reorganize the render tests

Move everything that might collect metrics to the metrics/ directory,
the only test manifests inside the render-test/ directory are the
unit tests of the probes.

circle.yml

@@ -36,8 +36,8 @@ workflows:
           target_is_linux: true
           config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8'
           install: true
-          memory_metrics: true
-          gfx_and_network_metrics: true
+          metrics: true
+          style_tests: true
       - next-build-template:
           name: next-linux-gcc4.9-release
           executor_name: ubuntu-disco
@@ -54,6 +54,7 @@ workflows:
           requires:
             - next-linux-gcc8-release
           config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 -DCMAKE_BUILD_TYPE=Debug -DMBGL_WITH_COVERAGE=ON'
+          style_tests: true
       - next-build-template:
           name: next-linux-clang8-release
           executor_name: ubuntu-disco
@@ -67,7 +68,8 @@ workflows:
             - next-macos-xcode11-release
           config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-8 -DCMAKE_CXX_COMPILER=clang++-8'
           install: true
-          binary_size_metrics: true
+          binary_size: true
+          style_tests: true
       - next-build-template:
           name: next-FIXME-linux-asan
           executor_name: ubuntu-disco
@@ -76,6 +78,7 @@ workflows:
             - next-sanity-checks
           config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-8 -DCMAKE_CXX_COMPILER=clang++-8 -DMBGL_WITH_SANITIZER=address'
           test_params: '|| true'
+          style_tests: true
       - next-build-template:
           name: next-linux-tsan
           executor_name: ubuntu-disco
@@ -83,6 +86,7 @@ workflows:
           requires:
             - next-sanity-checks
           config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-8 -DCMAKE_CXX_COMPILER=clang++-8 -DMBGL_WITH_SANITIZER=thread'
+          style_tests: true
       - next-build-template:
           name: next-FIXME-linux-valgrind
           executor_name: ubuntu-disco
@@ -97,6 +101,7 @@ workflows:
             - next-sanity-checks
           config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-8 -DCMAKE_CXX_COMPILER=clang++-8 -DMBGL_WITH_SANITIZER=undefined'
           test_params: '|| true'
+          style_tests: true
       - next-build-template:
           name: next-qt5-linux-gcc5-release
           executor_name: ubuntu-disco
@@ -116,6 +121,7 @@ workflows:
           target_is_macos: true
           config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_CXX_COMPILER_LAUNCHER=ccache'
           install: true
+          style_tests: true
       - next-build-template:
           name: next-macos-xcode11-debug
           executor_name: macos-11_0_0
@@ -123,6 +129,7 @@ workflows:
           requires:
             - next-macos-xcode11-release
           config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_BUILD_TYPE=Debug'
+          style_tests: true
       - next-build-template:
           name: next-ios-xcode11-release
           executor_name: macos-11_0_0
@@ -189,7 +196,7 @@ executors:
     environment:
       DISPLAY: :99
       UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=0
-      ASAN_OPTIONS: strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1
+      ASAN_OPTIONS: strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:halt_on_error=0
       QT_INSTALL_DOCS: /usr/share/qt5/doc
       QT_VERSION: 5
   macos-11_0_0:
@@ -752,13 +759,13 @@ jobs:
       install:
         type: boolean
         default: false
-      binary_size_metrics:
+      binary_size:
         type: boolean
         default: false
-      memory_metrics:
+      metrics:
         type: boolean
         default: false
-      gfx_and_network_metrics:
+      style_tests:
         type: boolean
         default: false
     executor: << parameters.executor_name >>
@@ -797,25 +804,25 @@ jobs:
           steps:
             - next-install
       - when:
-          condition: << parameters.binary_size_metrics >>
+          condition: << parameters.style_tests >>
+          steps:
+            - next-metrics:
+                step_name: 'Style Tests'
+                metrics_params: '-u rebaseline -p metrics/$CIRCLE_JOB-style.json'
+      - when:
+          condition: << parameters.metrics >>
+          steps:
+            - next-metrics:
+                step_name: 'Metrics'
+                metrics_params: '-u rebaseline -p metrics/$CIRCLE_JOB-metrics.json'
+      - when:
+          condition: << parameters.binary_size >>
           steps:
             - attach_workspace:
                 at: workspace
             - next-metrics:
-                step_name: 'Binary Size Metrics'
-                metrics_params: '-p metrics/binary-size.json'
-      - when:
-          condition: << parameters.memory_metrics >>
-          steps:
-            - next-metrics:
-                step_name: 'Memory Metrics'
-                metrics_params: 'render-tests -u rebaseline -p metrics/$CIRCLE_JOB-probe-memory.json'
-      - when:
-          condition: << parameters.gfx_and_network_metrics >>
-          steps:
-            - next-metrics:
-                step_name: 'GFX and Network Metrics'
-                metrics_params: 'render-tests -u rebaseline -p metrics/$CIRCLE_JOB-probe-gfx-network.json'
+                step_name: 'Binary Size'
+                metrics_params: '-u rebaseline -p metrics/binary-size.json'
       - next-save
 # ------------------------------------------------------------------------------
   node-linux-release:

expression-test/expression_test_parser.cpp

@@ -360,7 +360,7 @@ std::tuple<filesystem::path, std::vector<filesystem::path>, bool, uint32_t> pars
 
 Ignores parseExpressionIgnores() {
     Ignores ignores;
-    const auto mainIgnoresPath = filesystem::path(TEST_RUNNER_ROOT_PATH).append("platform/node/test/ignores.json");
+    const auto mainIgnoresPath = filesystem::path(TEST_RUNNER_ROOT_PATH).append("metrics/ignores/platform-all.json");
     auto maybeIgnores = readJson(mainIgnoresPath);
     if (!maybeIgnores.is<JSDocument>()) { // NOLINT
         return {};

metrics/ignores/platform-all.json

@@ -0,0 +1,133 @@
+{
+    "probes/memory/pass-memory-size-is-same": "TODO: Check with Mikhail why is this failing",
+    "binary-size/android-arm64-v8a": "skip - only makes sense for the binary size bot.",
+    "binary-size/android-armeabi-v7a": "skip - only makes sense for the binary size bot.",
+    "binary-size/android-x86": "skip - only makes sense for the binary size bot.",
+    "binary-size/android-x86_64": "skip - only makes sense for the binary size bot.",
+    "binary-size/linux-clang8": "skip - only makes sense for the binary size bot.",
+    "binary-size/linux-gcc8": "skip - only makes sense for the binary size bot.",
+    "binary-size/macos-xcode11": "skip - only makes sense for the binary size bot.",
+    "expression-tests/collator/accent-equals-de": "Locale-specific behavior changes based on platform.",
+    "expression-tests/in/assert-array": "https://github.com/mapbox/mapbox-gl-native/issues/15893",
+    "expression-tests/in/assert-string": "https://github.com/mapbox/mapbox-gl-native/issues/15893",
+    "expression-tests/in/basic-array": "https://github.com/mapbox/mapbox-gl-native/issues/15893",
+    "expression-tests/in/basic-string": "https://github.com/mapbox/mapbox-gl-native/issues/15893",
+    "expression-tests/in/invalid-haystack": "https://github.com/mapbox/mapbox-gl-native/issues/15893",
+    "expression-tests/in/invalid-needle": "https://github.com/mapbox/mapbox-gl-native/issues/15893",
+    "expression-tests/interpolate-hcl/linear": "https://github.com/mapbox/mapbox-gl-native/issues/8720",
+    "expression-tests/interpolate-lab/linear": "https://github.com/mapbox/mapbox-gl-native/issues/8720",
+    "expression-tests/is-supported-script/default": "This tests RTL text plugin behavior specific to GL JS",
+    "expression-tests/legacy/categorical/array": "https://github.com/mapbox/mapbox-gl-native/issues/12747",
+    "expression-tests/legacy/categorical/boolean": "https://github.com/mapbox/mapbox-gl-native/issues/12747",
+    "expression-tests/legacy/categorical/color": "https://github.com/mapbox/mapbox-gl-native/issues/12747",
+    "expression-tests/legacy/categorical/number": "https://github.com/mapbox/mapbox-gl-native/issues/12747",
+    "expression-tests/legacy/categorical/string": "https://github.com/mapbox/mapbox-gl-native/issues/12747",
+    "expression-tests/legacy/exponential/color-hcl": "https://github.com/mapbox/mapbox-gl-native/issues/12747",
+    "expression-tests/legacy/exponential/color-lab": "https://github.com/mapbox/mapbox-gl-native/issues/12747",
+    "expression-tests/legacy/identity/color-default": "https://github.com/mapbox/mapbox-gl-native/issues/12747",
+    "expression-tests/legacy/identity/enum-default": "https://github.com/mapbox/mapbox-gl-native/issues/12747",
+    "expression-tests/legacy/interval/composite": "https://github.com/mapbox/mapbox-gl-native/issues/12747",
+    "expression-tests/legacy/interval/composite-default": "https://github.com/mapbox/mapbox-gl-native/issues/12747",
+    "expression-tests/legacy/interval/tokens-zoom": "https://github.com/mapbox/mapbox-gl-native/issues/12747",
+    "expression-tests/resolved-locale/basic": "Even the 'en' locale may not be present on some test systems.",
+    "probes/file-size/fail-file-doesnt-match": "Should fail, doesn't match the expectation.",
+    "probes/file-size/fail-file-not-found": "Should fail, file not found.",
+    "probes/file-size/fail-size-is-over": "Should fail, size is bigger than expected.",
+    "probes/file-size/fail-size-is-under": "Should fail, size is smaller than expected.",
+    "probes/gfx/fail-ib-mem-mismatch": "Should fail, combined byte size of index buffers doesn't match the expectation.",
+    "probes/gfx/fail-negative-framebuffer-count": "Should fail, number of frame buffers is higher than expected.",
+    "probes/gfx/fail-texture-mem-mismatch": "Should fail, combined byte size of textures doesn't match the expectation.",
+    "probes/gfx/fail-too-few-buffers": "Should fail, number of vertex and index buffers is smaller than expected.",
+    "probes/gfx/fail-too-few-textures": "Should fail, number of textures is smaller than expected.",
+    "probes/gfx/fail-too-many-drawcalls": "Should fail, number of draw calls higher than expected.",
+    "probes/gfx/fail-vb-mem-mismatch": "Should fail, combined byte size of index buffers doesn't match the expectation.",
+    "probes/memory/fail-memory-size-is-too-big": "Should fail, memory size is bigger than expected.",
+    "probes/memory/fail-memory-size-is-too-small": "Should fail, memory size is smaller than expected.",
+    "probes/network/fail-requests": "Should fail, number of requests higher than expected.",
+    "probes/network/fail-requests-transferred": "Should fail, number of requests higher than expected and amount of transferred data less than expected.",
+    "probes/network/fail-transferred": "Should fail, amount of transferred data higher than expected.",
+    "query-tests/fill-extrusion-translate/multiple-layers": "https://github.com/mapbox/mapbox-gl-native/issues/12701",
+    "query-tests/fill-extrusion/base-in": "https://github.com/mapbox/mapbox-gl-native/issues/13139",
+    "query-tests/fill-extrusion/box-in": "https://github.com/mapbox/mapbox-gl-native/issues/13139",
+    "query-tests/fill-extrusion/side-in": "https://github.com/mapbox/mapbox-gl-native/issues/13139",
+    "query-tests/fill-extrusion/sort": "https://github.com/mapbox/mapbox-gl-native/issues/13139",
+    "query-tests/fill-extrusion/sort-concave-inner": "https://github.com/mapbox/mapbox-gl-native/issues/13139",
+    "query-tests/fill-extrusion/sort-concave-outer": "https://github.com/mapbox/mapbox-gl-native/issues/13139",
+    "query-tests/fill-extrusion/sort-rotated": "https://github.com/mapbox/mapbox-gl-native/issues/13139",
+    "query-tests/fill-extrusion/top-in": "https://github.com/mapbox/mapbox-gl-native/issues/13139",
+    "query-tests/fill-translate/multiple-layers": "https://github.com/mapbox/mapbox-gl-native/issues/12701",
+    "query-tests/geometry/multilinestring": "needs investigation",
+    "query-tests/geometry/multipolygon": "needs investigation",
+    "query-tests/geometry/polygon": "needs investigation",
+    "query-tests/regressions/mapbox-gl-js#6555": "skip - no querySourceFeatures in mbgl-node; needs issue",
+    "query-tests/regressions/mapbox-gl-js#7883": "https://github.com/mapbox/mapbox-gl-native/issues/14585",
+    "query-tests/world-wrapping/box": "skip - needs issue",
+    "render-tests/background-color/colorSpace-hcl": "needs issue",
+    "render-tests/background-color/transition": "https://github.com/mapbox/mapbox-gl-native/issues/10619",
+    "render-tests/canvas/default": "skip - js specific",
+    "render-tests/canvas/update": "skip - js specific",
+    "render-tests/circle-sort-key/literal": "https://github.com/mapbox/mapbox-gl-native/issues/15008",
+    "render-tests/collator/default": "Some test platforms don't resolve 'en' locale",
+    "render-tests/collator/resolved-locale": "Some test platforms don't resolve 'en' locale",
+    "render-tests/custom-layer-js/depth": "skip - js specific",
+    "render-tests/custom-layer-js/null-island": "skip - js specific",
+    "render-tests/custom-layer-js/tent-3d": "skip - js specific",
+    "render-tests/debug/collision": "https://github.com/mapbox/mapbox-gl-native/issues/3841",
+    "render-tests/debug/overdraw": "https://github.com/mapbox/mapbox-gl-native/issues/15638",
+    "render-tests/debug/raster": "https://github.com/mapbox/mapbox-gl-native/issues/15510",
+    "render-tests/debug/tile": "https://github.com/mapbox/mapbox-gl-native/issues/3841",
+    "render-tests/debug/tile-overscaled": "https://github.com/mapbox/mapbox-gl-native/issues/3841",
+    "render-tests/extent/1024-circle": "needs investigation",
+    "render-tests/feature-state/promote-id": "https://github.com/mapbox/mapbox-gl-native/issues/16016",
+    "render-tests/fill-extrusion-geometry/linestring": "https://github.com/mapbox/mapbox-gl-native/pull/14240",
+    "render-tests/fill-extrusion-pattern/@2x": "https://github.com/mapbox/mapbox-gl-js/issues/3327",
+    "render-tests/fill-extrusion-pattern/feature-expression": "https://github.com/mapbox/mapbox-gl-js/issues/3327",
+    "render-tests/fill-extrusion-pattern/function": "https://github.com/mapbox/mapbox-gl-js/issues/3327",
+    "render-tests/fill-extrusion-pattern/function-2": "https://github.com/mapbox/mapbox-gl-js/issues/3327",
+    "render-tests/fill-extrusion-pattern/literal": "https://github.com/mapbox/mapbox-gl-js/issues/3327",
+    "render-tests/fill-extrusion-pattern/opacity": "https://github.com/mapbox/mapbox-gl-js/issues/3327",
+    "render-tests/fill-extrusion-pattern/tile-buffer": "https://github.com/mapbox/mapbox-gl-js/issues/3327",
+    "render-tests/fill-pattern/update-feature-state": "https://github.com/mapbox/mapbox-gl-native/issues/15895",
+    "render-tests/fill-sort-key/literal": "https://github.com/mapbox/mapbox-gl-native/issues/15008",
+    "render-tests/geojson/inline-linestring-fill": "current behavior is arbitrary",
+    "render-tests/icon-text-fit/stretch-fifteen-part": "https://github.com/mapbox/mapbox-gl-native/issues/16017",
+    "render-tests/icon-text-fit/stretch-nine-part": "https://github.com/mapbox/mapbox-gl-native/issues/16017",
+    "render-tests/icon-text-fit/stretch-nine-part-@2x": "https://github.com/mapbox/mapbox-gl-native/issues/16017",
+    "render-tests/icon-text-fit/stretch-nine-part-content": "https://github.com/mapbox/mapbox-gl-native/issues/16017",
+    "render-tests/icon-text-fit/stretch-nine-part-content-collision": "https://github.com/mapbox/mapbox-gl-native/issues/16017",
+    "render-tests/icon-text-fit/stretch-nine-part-just-height": "https://github.com/mapbox/mapbox-gl-native/issues/16017",
+    "render-tests/icon-text-fit/stretch-nine-part-just-width": "https://github.com/mapbox/mapbox-gl-native/issues/16017",
+    "render-tests/icon-text-fit/stretch-three-part": "https://github.com/mapbox/mapbox-gl-native/issues/16017",
+    "render-tests/icon-text-fit/stretch-two-part": "https://github.com/mapbox/mapbox-gl-native/issues/16017",
+    "render-tests/icon-text-fit/stretch-underscale": "https://github.com/mapbox/mapbox-gl-native/issues/16017",
+    "render-tests/icon-text-fit/text-variable-anchor-overlap": "https://github.com/mapbox/mapbox-gl-native/issues/15809",
+    "render-tests/line-sort-key/literal": "https://github.com/mapbox/mapbox-gl-native/issues/15008",
+    "render-tests/mixed-zoom/z10-z11": "https://github.com/mapbox/mapbox-gl-native/issues/10397",
+    "render-tests/raster-masking/overlapping-zoom": "https://github.com/mapbox/mapbox-gl-native/issues/10195",
+    "render-tests/real-world/bangkok": "https://github.com/mapbox/mapbox-gl-native/issues/10412",
+    "render-tests/real-world/chicago": "https://github.com/mapbox/mapbox-gl-native/issues/10412",
+    "render-tests/real-world/sanfrancisco": "https://github.com/mapbox/mapbox-gl-native/issues/10412",
+    "render-tests/regressions/mapbox-gl-js#2467": "https://github.com/mapbox/mapbox-gl-native/issues/10619",
+    "render-tests/regressions/mapbox-gl-js#2762": "https://github.com/mapbox/mapbox-gl-native/issues/10619",
+    "render-tests/regressions/mapbox-gl-js#2769": "https://github.com/mapbox/mapbox-gl-native/issues/10619",
+    "render-tests/regressions/mapbox-gl-js#5740": "https://github.com/mapbox/mapbox-gl-native/issues/10619",
+    "render-tests/regressions/mapbox-gl-js#5982": "https://github.com/mapbox/mapbox-gl-native/issues/10619",
+    "render-tests/regressions/mapbox-gl-js#6655": "skip - port https://github.com/mapbox/mapbox-gl-js/pull/6263 - needs issue",
+    "render-tests/regressions/mapbox-gl-js#6706": "https://github.com/mapbox/mapbox-gl-native/issues/10619",
+    "render-tests/regressions/mapbox-gl-js#6806": "pending https://github.com/mapbox/mapbox-gl-js/pull/6812",
+    "render-tests/regressions/mapbox-gl-js#7271": "https://github.com/mapbox/mapbox-gl-native/issues/12888",
+    "render-tests/regressions/mapbox-gl-js#7302": "skip - js specific",
+    "render-tests/regressions/mapbox-gl-js#7708": "skip - js specific",
+    "render-tests/regressions/mapbox-gl-js#8026": "skip - js specific",
+    "render-tests/regressions/mapbox-gl-js#8817": "skip - https://github.com/mapbox/mapbox-gl-native/issues/15737",
+    "render-tests/regressions/mapbox-gl-js#9009": "https://github.com/mapbox/mapbox-gl-native/issues/16018",
+    "render-tests/regressions/mapbox-gl-native#7357": "https://github.com/mapbox/mapbox-gl-native/issues/7357",
+    "render-tests/runtime-styling/image-add-remove-add": "skip - https://github.com/mapbox/mapbox-gl-native/issues/16021",
+    "render-tests/runtime-styling/pattern-add-remove-add": "skip - https://github.com/mapbox/mapbox-gl-native/issues/16021",
+    "render-tests/symbol-cross-fade/chinese": "https://github.com/mapbox/mapbox-gl-native/issues/10619",
+    "render-tests/text-max-width/zero-width-point-placement": "https://github.com/mapbox/mapbox-gl-native/issues/15648",
+    "render-tests/text-size/nan": "https://github.com/mapbox/mapbox-gl-native/issues/16020",
+    "render-tests/text-variable-anchor/remember-last-placement": "skip - fails on gl-native, as symbol index is not functional at static map mode - needs issue",
+    "render-tests/video/default": "skip - https://github.com/mapbox/mapbox-gl-native/issues/601",
+    "render-tests/zoomed-fill/negative-zoom": "https://github.com/mapbox/mapbox-gl-native/issues/16019"
+}

metrics/ignores/platform-android.json

@@ -0,0 +1,13 @@
+{
+    "probes/gfx/pass": "Needs to be investigated and fixed.",
+    "probes/gfx/pass-double-probe": "Needs to be investigated and fixed.",
+    "probes/gfx/pass-probe-reset": "Needs to be investigated and fixed.",
+    "probes/network/pass": "Needs to be investigated and fixed.",
+    "render-tests/icon-text-fit/both-text-anchor-1x-image-2x-screen": "Temporarily fails as android test app does not derefence symbolinks in sprites folder.",
+    "render-tests/icon-text-fit/both-text-anchor-2x-image-1x-screen": "Temporarily fails as android test app does not derefence symbolinks in sprites folder.",
+    "render-tests/line-pattern/overscaled": "Needs to be investigated and fixed.",
+    "render-tests/regressions/mapbox-gl-js#5911a": "Needs to be investigated and fixed.",
+    "render-tests/regressions/mapbox-gl-native#9976": "Needs to be investigated and fixed.",
+    "render-tests/runtime-styling/image-add-pattern": "Needs to be investigated and fixed.",
+    "render-tests/runtime-styling/image-update-pattern": "Needs to be investigated and fixed."
+}

metrics/ignores/platform-linux.json

@@ -0,0 +1,10 @@
+{
+    "render-tests/fill-opacity/zoom-and-property-function-pattern": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/15322",
+    "render-tests/fill-pattern/literal": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/14423",
+    "render-tests/fill-pattern/opacity": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/14870",
+    "render-tests/fill-pattern/zoomed": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/14768",
+    "render-tests/line-pattern/opacity": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/15320",
+    "render-tests/line-translate/literal": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/14859",
+    "render-tests/regressions/mapbox-gl-js#5642": "Failing with mbgl-render-test",
+    "render-tests/regressions/mapbox-gl-js#7066": "Failing with mbgl-render-test"
+}

metrics/ignores/platform-macos.json

@@ -0,0 +1,4 @@
+{
+    "render-tests/regressions/mapbox-gl-js#5642": "Failing with mbgl-render-test",
+    "render-tests/regressions/mapbox-gl-js#7066": "Failing with mbgl-render-test"
+}

metrics/next-FIXME-linux-asan-style.json

@@ -0,0 +1,12 @@
+{
+    "asset_path": "../mapbox-gl-js/test/integration",
+    "base_test_path": "../mapbox-gl-js/test/integration",
+    "expectation_paths": [
+        "expectations/platform-all"
+    ],
+    "ignore_paths": [
+        "ignores/platform-all.json",
+        "ignores/platform-linux.json"
+    ],
+    "vendor_path": "../vendor"
+}

metrics/next-FIXME-linux-ubsan-style.json

@@ -0,0 +1,12 @@
+{
+    "asset_path": "../mapbox-gl-js/test/integration",
+    "base_test_path": "../mapbox-gl-js/test/integration",
+    "expectation_paths": [
+        "expectations/platform-all"
+    ],
+    "ignore_paths": [
+        "ignores/platform-all.json",
+        "ignores/platform-linux.json"
+    ],
+    "vendor_path": "../vendor"
+}

metrics/next-android-render-test-runner-metrics.json

@@ -0,0 +1,19 @@
+{
+    "base_test_path": "tests",
+    "expectation_paths": [
+        "expectations/platform-all",
+        "expectations/platform-android"
+    ],
+    "ignore_paths": [
+        "ignores/platform-all.json",
+        "ignores/platform-linux.json",
+        "ignores/platform-android.json"
+    ],
+    "metric_path": "next-android-render-test-runner",
+    "probes": [
+        "probeGFX",
+        "probeMemory",
+        "probeNetwork"
+    ],
+    "result_path": "/sdcard"
+}

metrics/next-android-render-test-runner-probe-gfx-network.json

@@ -1,10 +0,0 @@
-{
-    "base_test_path":"mapbox-gl-js/test/integration",
-    "expectation_paths":["render-test/expected", "render-test/expected/android"],
-    "ignore_paths":["render-test/ignores/ignores.json", "render-test/ignores/linux-ignores.json", "render-test/ignores/android-temp-fail.json"],
-    "vendor_path":"vendor",
-    "asset_path": "mapbox-gl-js/test/integration",
-    "result_path":"/sdcard/",
-    "probes": ["probeGFX", "probeNetwork"],
-    "metric_path":"metrics/next-android-render-test-runner"
-}  

metrics/next-android-render-test-runner-probe-memory.json

@@ -1,11 +0,0 @@
-{
-    "base_test_path":"mapbox-gl-js/test/integration",
-    "expectation_paths":["render-test/expected", "render-test/expected/android"],
-    "ignore_paths":["render-test/ignores/ignores.json", "render-test/ignores/linux-ignores.json", "render-test/ignores/android-temp-fail.json"],
-    "vendor_path":"vendor",
-    "asset_path": "mapbox-gl-js/test/integration",
-    "result_path":"/sdcard/",
-    "probes": ["probeMemory"],
-    "metric_path":"metrics/next-android-render-test-runner",
-    "filter": "real-world"
-}  

metrics/next-android-render-test-runner-style.json

@@ -0,0 +1,20 @@
+{
+    "asset_path": "../mapbox-gl-js/test/integration",
+    "base_test_path": "../mapbox-gl-js/test/integration",
+    "expectation_paths": [
+        "expectations/platform-all",
+        "expectations/platform-android"
+    ],
+    "ignore_paths": [
+        "ignores/platform-all.json",
+        "ignores/platform-linux.json",
+        "ignores/platform-android.json"
+    ],
+    "metric_path": "next-android-render-test-runner",
+    "probes": [
+        "probeGFX",
+        "probeNetwork"
+    ],
+    "result_path": "/sdcard",
+    "vendor_path": "../vendor"
+}

metrics/next-linux-clang8-release-style.json

@@ -0,0 +1,12 @@
+{
+    "asset_path": "../mapbox-gl-js/test/integration",
+    "base_test_path": "../mapbox-gl-js/test/integration",
+    "expectation_paths": [
+        "expectations/platform-all"
+    ],
+    "ignore_paths": [
+        "ignores/platform-all.json",
+        "ignores/platform-linux.json"
+    ],
+    "vendor_path": "../vendor"
+}

metrics/next-linux-gcc8-debug-coverage-style.json

@@ -0,0 +1,12 @@
+{
+    "asset_path": "../mapbox-gl-js/test/integration",
+    "base_test_path": "../mapbox-gl-js/test/integration",
+    "expectation_paths": [
+        "expectations/platform-all"
+    ],
+    "ignore_paths": [
+        "ignores/platform-all.json",
+        "ignores/platform-linux.json"
+    ],
+    "vendor_path": "../vendor"
+}

metrics/next-linux-gcc8-release-metrics.json

@@ -0,0 +1,18 @@
+{
+    "asset_path": "../mapbox-gl-js/test/integration",
+    "base_test_path": "tests",
+    "expectation_paths": [
+        "expectations/platform-all"
+    ],
+    "ignore_paths": [
+        "ignores/platform-all.json",
+        "ignores/platform-linux.json"
+    ],
+    "metric_path": "next-linux-gcc8-release",
+    "probes": [
+        "probeGFX",
+        "probeMemory",
+        "probeNetwork"
+    ],
+    "vendor_path": "../vendor"
+}

metrics/next-linux-gcc8-release-probe-gfx-network.json

@@ -1,9 +0,0 @@
-{
-    "base_test_path":"../mapbox-gl-js/test/integration",
-    "expectation_paths":["../render-test/expected/"],
-    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"],
-    "vendor_path":"../vendor",
-    "asset_path": "../mapbox-gl-js/test/integration",
-    "probes": ["probeGFX", "probeNetwork"],
-    "metric_path":"../metrics/next-linux-gcc8-release"
-}

metrics/next-linux-gcc8-release-probe-memory.json

@@ -1,10 +0,0 @@
-{
-    "base_test_path":"../mapbox-gl-js/test/integration",
-    "expectation_paths":["../render-test/expected/"],
-    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"],
-    "vendor_path":"../vendor",
-    "asset_path": "../mapbox-gl-js/test/integration",
-    "probes": ["probeMemory"],
-    "metric_path":"../metrics/next-linux-gcc8-release",
-    "filter": "real-world"
-}

metrics/next-linux-gcc8-release-style.json

@@ -0,0 +1,17 @@
+{
+    "asset_path": "../mapbox-gl-js/test/integration",
+    "base_test_path": "../mapbox-gl-js/test/integration",
+    "expectation_paths": [
+        "expectations/platform-all"
+    ],
+    "ignore_paths": [
+        "ignores/platform-all.json",
+        "ignores/platform-linux.json"
+    ],
+    "metric_path": "next-linux-gcc8-release",
+    "probes": [
+        "probeGFX",
+        "probeNetwork"
+    ],
+    "vendor_path": "../vendor"
+}

metrics/next-linux-tsan-style.json

@@ -0,0 +1,12 @@
+{
+    "asset_path": "../mapbox-gl-js/test/integration",
+    "base_test_path": "../mapbox-gl-js/test/integration",
+    "expectation_paths": [
+        "expectations/platform-all"
+    ],
+    "ignore_paths": [
+        "ignores/platform-all.json",
+        "ignores/platform-linux.json"
+    ],
+    "vendor_path": "../vendor"
+}

metrics/next-macos-xcode11-debug-style.json

@@ -0,0 +1,12 @@
+{
+    "asset_path": "../mapbox-gl-js/test/integration",
+    "base_test_path": "../mapbox-gl-js/test/integration",
+    "expectation_paths": [
+        "expectations/platform-all"
+    ],
+    "ignore_paths": [
+        "ignores/platform-all.json",
+        "ignores/platform-macos.json"
+    ],
+    "vendor_path": "../vendor"
+}

metrics/next-macos-xcode11-release-style.json

@@ -0,0 +1,12 @@
+{
+    "asset_path": "../mapbox-gl-js/test/integration",
+    "base_test_path": "../mapbox-gl-js/test/integration",
+    "expectation_paths": [
+        "expectations/platform-all"
+    ],
+    "ignore_paths": [
+        "ignores/platform-all.json",
+        "ignores/platform-macos.json"
+    ],
+    "vendor_path": "../vendor"
+}