From 970cabe0eb441270ca3af44e495a5fee997ab54c Mon Sep 17 00:00:00 2001 From: "Thiago Marcos P. Santos" Date: Tue, 10 Dec 2019 17:10:48 +0200 Subject: [PATCH] [build] Reorganize the render tests Move everything that might collect metrics to the metrics/ directory, the only test manifests inside the render-test/ directory are the unit tests of the probes. --- circle.yml | 51 ++++--- expression-test/expression_test_parser.cpp | 2 +- .../collision-lines-overscaled/expected.png | Bin .../collision-lines-pitched/expected.png | Bin .../debug/collision-lines/expected.png | Bin .../collision-pitched-wrapped/expected.png | Bin .../symbol-visibility/visible/expected.png | Bin .../expected.png | Bin .../expected.png | Bin .../expected.png | Bin .../pitched-rotated-debug/expected.png | Bin .../rotated-offset/expected.png | Bin .../expected.png | Bin .../mapbox-gl-js#2305/expected.png | Bin .../mapbox-gl-js#5978/expected.png | Bin .../sprites/1x-screen-1x-pattern/expected.png | Bin .../sprites/1x-screen-2x-pattern/expected.png | Bin .../sprites/2x-screen-1x-pattern/expected.png | Bin .../sprites/2x-screen-2x-pattern/expected.png | Bin metrics/ignores/platform-all.json | 133 ++++++++++++++++++ metrics/ignores/platform-android.json | 13 ++ metrics/ignores/platform-linux.json | 10 ++ metrics/ignores/platform-macos.json | 4 + metrics/next-FIXME-linux-asan-style.json | 12 ++ metrics/next-FIXME-linux-ubsan-style.json | 12 ++ ...xt-android-render-test-runner-metrics.json | 19 +++ ...-render-test-runner-probe-gfx-network.json | 10 -- ...droid-render-test-runner-probe-memory.json | 11 -- ...next-android-render-test-runner-style.json | 20 +++ metrics/next-linux-clang8-release-style.json | 12 ++ .../next-linux-gcc8-debug-coverage-style.json | 12 ++ metrics/next-linux-gcc8-release-metrics.json | 18 +++ ...-linux-gcc8-release-probe-gfx-network.json | 9 -- .../next-linux-gcc8-release-probe-memory.json | 10 -- metrics/next-linux-gcc8-release-style.json | 17 +++ metrics/next-linux-tsan-style.json | 12 ++ metrics/next-macos-xcode11-debug-style.json | 12 ++ metrics/next-macos-xcode11-release-style.json | 12 ++ .../fail-file-doesnt-match/expected.png | Bin .../fail-file-doesnt-match/metrics.json | 0 .../fail-file-doesnt-match/style.json | 0 .../fail-file-not-found/expected.png | Bin .../file-size/fail-file-not-found/style.json | 0 .../file-size/fail-size-is-over/expected.png | Bin .../file-size/fail-size-is-over/metrics.json | 0 .../file-size/fail-size-is-over/style.json | 0 .../file-size/fail-size-is-under/expected.png | Bin .../file-size/fail-size-is-under/metrics.json | 0 .../file-size/fail-size-is-under/style.json | 0 .../expected.png | Bin .../metrics.json | 0 .../style.json | 0 .../expected.png | Bin .../metrics.json | 0 .../style.json | 0 .../file-size/pass-size-is-same/expected.png | Bin .../file-size/pass-size-is-same/metrics.json | 0 .../file-size/pass-size-is-same/style.json | 0 .../gfx/fail-ib-mem-mismatch/expected.png | Bin .../gfx/fail-ib-mem-mismatch/metrics.json | 0 .../gfx/fail-ib-mem-mismatch/style.json | 0 .../expected.png | Bin .../metrics.json | 0 .../style.json | 0 .../fail-texture-mem-mismatch/expected.png | Bin .../fail-texture-mem-mismatch/metrics.json | 0 .../gfx/fail-texture-mem-mismatch/style.json | 0 .../gfx/fail-too-few-buffers/expected.png | Bin .../gfx/fail-too-few-buffers/metrics.json | 0 .../gfx/fail-too-few-buffers/style.json | 0 .../gfx/fail-too-few-textures/expected.png | Bin .../gfx/fail-too-few-textures/metrics.json | 0 .../gfx/fail-too-few-textures/style.json | 0 .../gfx/fail-too-many-drawcalls/expected.png | Bin .../gfx/fail-too-many-drawcalls/metrics.json | 0 .../gfx/fail-too-many-drawcalls/style.json | 0 .../gfx/fail-vb-mem-mismatch/expected.png | Bin .../gfx/fail-vb-mem-mismatch/metrics.json | 0 .../gfx/fail-vb-mem-mismatch/style.json | 0 .../gfx/pass-double-probe/expected.png | Bin .../gfx/pass-double-probe/metrics.json | 0 .../probes}/gfx/pass-double-probe/style.json | 0 .../probes}/gfx/pass-probe-reset/expected.png | Bin .../probes}/gfx/pass-probe-reset/metrics.json | 0 .../probes}/gfx/pass-probe-reset/style.json | 0 .../tests/probes}/gfx/pass/expected.png | Bin .../tests/probes}/gfx/pass/metrics.json | 0 .../tests/probes}/gfx/pass/style.json | 0 .../fail-memory-size-is-too-big}/expected.png | Bin .../fail-memory-size-is-too-big/metrics.json | 0 .../fail-memory-size-is-too-big/style.json | 0 .../expected.png | Bin .../metrics.json | 0 .../fail-memory-size-is-too-small/style.json | 0 .../pass-memory-size-is-same}/expected.png | Bin .../pass-memory-size-is-same/metrics.json | 0 .../pass-memory-size-is-same/style.json | 0 .../fail-requests-transferred/expected.png | Bin .../fail-requests-transferred/metrics.json | 0 .../fail-requests-transferred/style.json | 0 .../network/fail-requests/expected.png | Bin .../network/fail-requests/metrics.json | 0 .../probes}/network/fail-requests/style.json | 0 .../network/fail-transferred/expected.png | Bin .../network/fail-transferred/metrics.json | 0 .../network/fail-transferred/style.json | 0 .../tests/probes}/network/pass/expected.png | Bin .../tests/probes}/network/pass/metrics.json | 0 .../tests/probes}/network/pass/style.json | 0 next/platform/android/android.cmake | 23 --- next/platform/linux/linux.cmake | 20 --- next/platform/macos/macos.cmake | 17 --- next/platform/node/CMakeLists.txt | 1 - .../android/src/test/render_test_runner.cpp | 4 +- platform/node/test/ignores.json | 109 -------------- .../android/app/src/main/assets/to_zip.txt | 9 +- render-test/ignores/android-temp-fail.json | 9 -- render-test/ignores/linux-ignores.json | 11 -- render-test/linux-manifest.json | 7 - render-test/linux-probe-manifest.json | 7 - render-test/linux-query-manifest.json | 6 - render-test/mac-ignores.json | 5 - render-test/mac-manifest.json | 7 - render-test/mac-probe-manifest.json | 8 -- .../pass-memory-size-is-same/metrics.json | 19 --- .../pass-memory-size-is-same/expected.png | Bin 96 -> 0 bytes render-test/tests/should-fail.json | 18 --- 127 files changed, 355 insertions(+), 336 deletions(-) rename {render-test/expected => metrics/expectations/platform-all}/render-tests/debug/collision-lines-overscaled/expected.png (100%) rename {render-test/expected => metrics/expectations/platform-all}/render-tests/debug/collision-lines-pitched/expected.png (100%) rename {render-test/expected => metrics/expectations/platform-all}/render-tests/debug/collision-lines/expected.png (100%) rename {render-test/expected => metrics/expectations/platform-all}/render-tests/debug/collision-pitched-wrapped/expected.png (100%) rename {render-test/expected => metrics/expectations/platform-all}/render-tests/symbol-visibility/visible/expected.png (100%) rename {render-test/expected => metrics/expectations/platform-all}/render-tests/text-pitch-alignment/auto-text-rotation-alignment-map/expected.png (100%) rename {render-test/expected => metrics/expectations/platform-all}/render-tests/text-pitch-alignment/map-text-rotation-alignment-map/expected.png (100%) rename {render-test/expected => metrics/expectations/platform-all}/render-tests/text-pitch-alignment/viewport-text-rotation-alignment-map/expected.png (100%) rename {render-test/expected => metrics/expectations/platform-all}/render-tests/text-variable-anchor/pitched-rotated-debug/expected.png (100%) rename {render-test/expected => metrics/expectations/platform-all}/render-tests/text-variable-anchor/rotated-offset/expected.png (100%) rename {render-test/expected/android => metrics/expectations/platform-android}/render-tests/combinations/symbol-translucent--fill-translucent/expected.png (100%) rename {render-test/expected/android => metrics/expectations/platform-android}/render-tests/regressions/mapbox-gl-js#2305/expected.png (100%) rename {render-test/expected/android => metrics/expectations/platform-android}/render-tests/regressions/mapbox-gl-js#5978/expected.png (100%) rename {render-test/expected/android => metrics/expectations/platform-android}/render-tests/sprites/1x-screen-1x-pattern/expected.png (100%) rename {render-test/expected/android => metrics/expectations/platform-android}/render-tests/sprites/1x-screen-2x-pattern/expected.png (100%) rename {render-test/expected/android => metrics/expectations/platform-android}/render-tests/sprites/2x-screen-1x-pattern/expected.png (100%) rename {render-test/expected/android => metrics/expectations/platform-android}/render-tests/sprites/2x-screen-2x-pattern/expected.png (100%) create mode 100644 metrics/ignores/platform-all.json create mode 100644 metrics/ignores/platform-android.json create mode 100644 metrics/ignores/platform-linux.json create mode 100644 metrics/ignores/platform-macos.json create mode 100644 metrics/next-FIXME-linux-asan-style.json create mode 100644 metrics/next-FIXME-linux-ubsan-style.json create mode 100644 metrics/next-android-render-test-runner-metrics.json delete mode 100644 metrics/next-android-render-test-runner-probe-gfx-network.json delete mode 100644 metrics/next-android-render-test-runner-probe-memory.json create mode 100644 metrics/next-android-render-test-runner-style.json create mode 100644 metrics/next-linux-clang8-release-style.json create mode 100644 metrics/next-linux-gcc8-debug-coverage-style.json create mode 100644 metrics/next-linux-gcc8-release-metrics.json delete mode 100644 metrics/next-linux-gcc8-release-probe-gfx-network.json delete mode 100644 metrics/next-linux-gcc8-release-probe-memory.json create mode 100644 metrics/next-linux-gcc8-release-style.json create mode 100644 metrics/next-linux-tsan-style.json create mode 100644 metrics/next-macos-xcode11-debug-style.json create mode 100644 metrics/next-macos-xcode11-release-style.json rename {render-test/tests => metrics/tests/probes}/file-size/fail-file-doesnt-match/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/file-size/fail-file-doesnt-match/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/file-size/fail-file-doesnt-match/style.json (100%) rename {render-test/tests => metrics/tests/probes}/file-size/fail-file-not-found/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/file-size/fail-file-not-found/style.json (100%) rename {render-test/tests => metrics/tests/probes}/file-size/fail-size-is-over/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/file-size/fail-size-is-over/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/file-size/fail-size-is-over/style.json (100%) rename {render-test/tests => metrics/tests/probes}/file-size/fail-size-is-under/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/file-size/fail-size-is-under/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/file-size/fail-size-is-under/style.json (100%) rename {render-test/tests => metrics/tests/probes}/file-size/pass-size-is-in-tolerance-higher/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/file-size/pass-size-is-in-tolerance-higher/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/file-size/pass-size-is-in-tolerance-higher/style.json (100%) rename {render-test/tests => metrics/tests/probes}/file-size/pass-size-is-in-tolerance-lower/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/file-size/pass-size-is-in-tolerance-lower/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/file-size/pass-size-is-in-tolerance-lower/style.json (100%) rename {render-test/tests => metrics/tests/probes}/file-size/pass-size-is-same/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/file-size/pass-size-is-same/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/file-size/pass-size-is-same/style.json (100%) rename {render-test/tests => metrics/tests/probes}/gfx/fail-ib-mem-mismatch/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/gfx/fail-ib-mem-mismatch/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/gfx/fail-ib-mem-mismatch/style.json (100%) rename {render-test/tests => metrics/tests/probes}/gfx/fail-negative-framebuffer-count/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/gfx/fail-negative-framebuffer-count/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/gfx/fail-negative-framebuffer-count/style.json (100%) rename {render-test/tests => metrics/tests/probes}/gfx/fail-texture-mem-mismatch/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/gfx/fail-texture-mem-mismatch/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/gfx/fail-texture-mem-mismatch/style.json (100%) rename {render-test/tests => metrics/tests/probes}/gfx/fail-too-few-buffers/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/gfx/fail-too-few-buffers/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/gfx/fail-too-few-buffers/style.json (100%) rename {render-test/tests => metrics/tests/probes}/gfx/fail-too-few-textures/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/gfx/fail-too-few-textures/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/gfx/fail-too-few-textures/style.json (100%) rename {render-test/tests => metrics/tests/probes}/gfx/fail-too-many-drawcalls/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/gfx/fail-too-many-drawcalls/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/gfx/fail-too-many-drawcalls/style.json (100%) rename {render-test/tests => metrics/tests/probes}/gfx/fail-vb-mem-mismatch/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/gfx/fail-vb-mem-mismatch/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/gfx/fail-vb-mem-mismatch/style.json (100%) rename {render-test/tests => metrics/tests/probes}/gfx/pass-double-probe/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/gfx/pass-double-probe/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/gfx/pass-double-probe/style.json (100%) rename {render-test/tests => metrics/tests/probes}/gfx/pass-probe-reset/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/gfx/pass-probe-reset/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/gfx/pass-probe-reset/style.json (100%) rename {render-test/tests => metrics/tests/probes}/gfx/pass/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/gfx/pass/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/gfx/pass/style.json (100%) rename {render-test/expected/tests/memory/pass-memory-size-is-same => metrics/tests/probes/memory/fail-memory-size-is-too-big}/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/memory/fail-memory-size-is-too-big/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/memory/fail-memory-size-is-too-big/style.json (100%) rename {render-test/tests/memory/fail-memory-size-is-too-big => metrics/tests/probes/memory/fail-memory-size-is-too-small}/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/memory/fail-memory-size-is-too-small/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/memory/fail-memory-size-is-too-small/style.json (100%) rename {render-test/tests/memory/fail-memory-size-is-too-small => metrics/tests/probes/memory/pass-memory-size-is-same}/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/memory/pass-memory-size-is-same/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/memory/pass-memory-size-is-same/style.json (100%) rename {render-test/tests => metrics/tests/probes}/network/fail-requests-transferred/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/network/fail-requests-transferred/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/network/fail-requests-transferred/style.json (100%) rename {render-test/tests => metrics/tests/probes}/network/fail-requests/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/network/fail-requests/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/network/fail-requests/style.json (100%) rename {render-test/tests => metrics/tests/probes}/network/fail-transferred/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/network/fail-transferred/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/network/fail-transferred/style.json (100%) rename {render-test/tests => metrics/tests/probes}/network/pass/expected.png (100%) rename {render-test/tests => metrics/tests/probes}/network/pass/metrics.json (100%) rename {render-test/tests => metrics/tests/probes}/network/pass/style.json (100%) delete mode 100644 platform/node/test/ignores.json delete mode 100644 render-test/ignores/android-temp-fail.json delete mode 100644 render-test/ignores/linux-ignores.json delete mode 100644 render-test/linux-manifest.json delete mode 100644 render-test/linux-probe-manifest.json delete mode 100644 render-test/linux-query-manifest.json delete mode 100644 render-test/mac-ignores.json delete mode 100644 render-test/mac-manifest.json delete mode 100644 render-test/mac-probe-manifest.json delete mode 100644 render-test/metrics/mac/tests/memory/pass-memory-size-is-same/metrics.json delete mode 100644 render-test/tests/memory/pass-memory-size-is-same/expected.png delete mode 100644 render-test/tests/should-fail.json diff --git a/circle.yml b/circle.yml index d0e13275f..0324db0e4 100644 --- a/circle.yml +++ b/circle.yml @@ -36,8 +36,8 @@ workflows: target_is_linux: true config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8' install: true - memory_metrics: true - gfx_and_network_metrics: true + metrics: true + style_tests: true - next-build-template: name: next-linux-gcc4.9-release executor_name: ubuntu-disco @@ -54,6 +54,7 @@ workflows: requires: - next-linux-gcc8-release config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 -DCMAKE_BUILD_TYPE=Debug -DMBGL_WITH_COVERAGE=ON' + style_tests: true - next-build-template: name: next-linux-clang8-release executor_name: ubuntu-disco @@ -67,7 +68,8 @@ workflows: - next-macos-xcode11-release config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-8 -DCMAKE_CXX_COMPILER=clang++-8' install: true - binary_size_metrics: true + binary_size: true + style_tests: true - next-build-template: name: next-FIXME-linux-asan executor_name: ubuntu-disco @@ -76,6 +78,7 @@ workflows: - next-sanity-checks config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-8 -DCMAKE_CXX_COMPILER=clang++-8 -DMBGL_WITH_SANITIZER=address' test_params: '|| true' + style_tests: true - next-build-template: name: next-linux-tsan executor_name: ubuntu-disco @@ -83,6 +86,7 @@ workflows: requires: - next-sanity-checks config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-8 -DCMAKE_CXX_COMPILER=clang++-8 -DMBGL_WITH_SANITIZER=thread' + style_tests: true - next-build-template: name: next-FIXME-linux-valgrind executor_name: ubuntu-disco @@ -97,6 +101,7 @@ workflows: - next-sanity-checks config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-8 -DCMAKE_CXX_COMPILER=clang++-8 -DMBGL_WITH_SANITIZER=undefined' test_params: '|| true' + style_tests: true - next-build-template: name: next-qt5-linux-gcc5-release executor_name: ubuntu-disco @@ -116,6 +121,7 @@ workflows: target_is_macos: true config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_CXX_COMPILER_LAUNCHER=ccache' install: true + style_tests: true - next-build-template: name: next-macos-xcode11-debug executor_name: macos-11_0_0 @@ -123,6 +129,7 @@ workflows: requires: - next-macos-xcode11-release config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_BUILD_TYPE=Debug' + style_tests: true - next-build-template: name: next-ios-xcode11-release executor_name: macos-11_0_0 @@ -189,7 +196,7 @@ executors: environment: DISPLAY: :99 UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=0 - ASAN_OPTIONS: strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1 + ASAN_OPTIONS: strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:halt_on_error=0 QT_INSTALL_DOCS: /usr/share/qt5/doc QT_VERSION: 5 macos-11_0_0: @@ -752,13 +759,13 @@ jobs: install: type: boolean default: false - binary_size_metrics: + binary_size: type: boolean default: false - memory_metrics: + metrics: type: boolean default: false - gfx_and_network_metrics: + style_tests: type: boolean default: false executor: << parameters.executor_name >> @@ -797,25 +804,25 @@ jobs: steps: - next-install - when: - condition: << parameters.binary_size_metrics >> + condition: << parameters.style_tests >> + steps: + - next-metrics: + step_name: 'Style Tests' + metrics_params: '-u rebaseline -p metrics/$CIRCLE_JOB-style.json' + - when: + condition: << parameters.metrics >> + steps: + - next-metrics: + step_name: 'Metrics' + metrics_params: '-u rebaseline -p metrics/$CIRCLE_JOB-metrics.json' + - when: + condition: << parameters.binary_size >> steps: - attach_workspace: at: workspace - next-metrics: - step_name: 'Binary Size Metrics' - metrics_params: '-p metrics/binary-size.json' - - when: - condition: << parameters.memory_metrics >> - steps: - - next-metrics: - step_name: 'Memory Metrics' - metrics_params: 'render-tests -u rebaseline -p metrics/$CIRCLE_JOB-probe-memory.json' - - when: - condition: << parameters.gfx_and_network_metrics >> - steps: - - next-metrics: - step_name: 'GFX and Network Metrics' - metrics_params: 'render-tests -u rebaseline -p metrics/$CIRCLE_JOB-probe-gfx-network.json' + step_name: 'Binary Size' + metrics_params: '-u rebaseline -p metrics/binary-size.json' - next-save # ------------------------------------------------------------------------------ node-linux-release: diff --git a/expression-test/expression_test_parser.cpp b/expression-test/expression_test_parser.cpp index 15136f023..44f8bebf5 100644 --- a/expression-test/expression_test_parser.cpp +++ b/expression-test/expression_test_parser.cpp @@ -360,7 +360,7 @@ std::tuple, bool, uint32_t> pars Ignores parseExpressionIgnores() { Ignores ignores; - const auto mainIgnoresPath = filesystem::path(TEST_RUNNER_ROOT_PATH).append("platform/node/test/ignores.json"); + const auto mainIgnoresPath = filesystem::path(TEST_RUNNER_ROOT_PATH).append("metrics/ignores/platform-all.json"); auto maybeIgnores = readJson(mainIgnoresPath); if (!maybeIgnores.is()) { // NOLINT return {}; diff --git a/render-test/expected/render-tests/debug/collision-lines-overscaled/expected.png b/metrics/expectations/platform-all/render-tests/debug/collision-lines-overscaled/expected.png similarity index 100% rename from render-test/expected/render-tests/debug/collision-lines-overscaled/expected.png rename to metrics/expectations/platform-all/render-tests/debug/collision-lines-overscaled/expected.png diff --git a/render-test/expected/render-tests/debug/collision-lines-pitched/expected.png b/metrics/expectations/platform-all/render-tests/debug/collision-lines-pitched/expected.png similarity index 100% rename from render-test/expected/render-tests/debug/collision-lines-pitched/expected.png rename to metrics/expectations/platform-all/render-tests/debug/collision-lines-pitched/expected.png diff --git a/render-test/expected/render-tests/debug/collision-lines/expected.png b/metrics/expectations/platform-all/render-tests/debug/collision-lines/expected.png similarity index 100% rename from render-test/expected/render-tests/debug/collision-lines/expected.png rename to metrics/expectations/platform-all/render-tests/debug/collision-lines/expected.png diff --git a/render-test/expected/render-tests/debug/collision-pitched-wrapped/expected.png b/metrics/expectations/platform-all/render-tests/debug/collision-pitched-wrapped/expected.png similarity index 100% rename from render-test/expected/render-tests/debug/collision-pitched-wrapped/expected.png rename to metrics/expectations/platform-all/render-tests/debug/collision-pitched-wrapped/expected.png diff --git a/render-test/expected/render-tests/symbol-visibility/visible/expected.png b/metrics/expectations/platform-all/render-tests/symbol-visibility/visible/expected.png similarity index 100% rename from render-test/expected/render-tests/symbol-visibility/visible/expected.png rename to metrics/expectations/platform-all/render-tests/symbol-visibility/visible/expected.png diff --git a/render-test/expected/render-tests/text-pitch-alignment/auto-text-rotation-alignment-map/expected.png b/metrics/expectations/platform-all/render-tests/text-pitch-alignment/auto-text-rotation-alignment-map/expected.png similarity index 100% rename from render-test/expected/render-tests/text-pitch-alignment/auto-text-rotation-alignment-map/expected.png rename to metrics/expectations/platform-all/render-tests/text-pitch-alignment/auto-text-rotation-alignment-map/expected.png diff --git a/render-test/expected/render-tests/text-pitch-alignment/map-text-rotation-alignment-map/expected.png b/metrics/expectations/platform-all/render-tests/text-pitch-alignment/map-text-rotation-alignment-map/expected.png similarity index 100% rename from render-test/expected/render-tests/text-pitch-alignment/map-text-rotation-alignment-map/expected.png rename to metrics/expectations/platform-all/render-tests/text-pitch-alignment/map-text-rotation-alignment-map/expected.png diff --git a/render-test/expected/render-tests/text-pitch-alignment/viewport-text-rotation-alignment-map/expected.png b/metrics/expectations/platform-all/render-tests/text-pitch-alignment/viewport-text-rotation-alignment-map/expected.png similarity index 100% rename from render-test/expected/render-tests/text-pitch-alignment/viewport-text-rotation-alignment-map/expected.png rename to metrics/expectations/platform-all/render-tests/text-pitch-alignment/viewport-text-rotation-alignment-map/expected.png diff --git a/render-test/expected/render-tests/text-variable-anchor/pitched-rotated-debug/expected.png b/metrics/expectations/platform-all/render-tests/text-variable-anchor/pitched-rotated-debug/expected.png similarity index 100% rename from render-test/expected/render-tests/text-variable-anchor/pitched-rotated-debug/expected.png rename to metrics/expectations/platform-all/render-tests/text-variable-anchor/pitched-rotated-debug/expected.png diff --git a/render-test/expected/render-tests/text-variable-anchor/rotated-offset/expected.png b/metrics/expectations/platform-all/render-tests/text-variable-anchor/rotated-offset/expected.png similarity index 100% rename from render-test/expected/render-tests/text-variable-anchor/rotated-offset/expected.png rename to metrics/expectations/platform-all/render-tests/text-variable-anchor/rotated-offset/expected.png diff --git a/render-test/expected/android/render-tests/combinations/symbol-translucent--fill-translucent/expected.png b/metrics/expectations/platform-android/render-tests/combinations/symbol-translucent--fill-translucent/expected.png similarity index 100% rename from render-test/expected/android/render-tests/combinations/symbol-translucent--fill-translucent/expected.png rename to metrics/expectations/platform-android/render-tests/combinations/symbol-translucent--fill-translucent/expected.png diff --git a/render-test/expected/android/render-tests/regressions/mapbox-gl-js#2305/expected.png b/metrics/expectations/platform-android/render-tests/regressions/mapbox-gl-js#2305/expected.png similarity index 100% rename from render-test/expected/android/render-tests/regressions/mapbox-gl-js#2305/expected.png rename to metrics/expectations/platform-android/render-tests/regressions/mapbox-gl-js#2305/expected.png diff --git a/render-test/expected/android/render-tests/regressions/mapbox-gl-js#5978/expected.png b/metrics/expectations/platform-android/render-tests/regressions/mapbox-gl-js#5978/expected.png similarity index 100% rename from render-test/expected/android/render-tests/regressions/mapbox-gl-js#5978/expected.png rename to metrics/expectations/platform-android/render-tests/regressions/mapbox-gl-js#5978/expected.png diff --git a/render-test/expected/android/render-tests/sprites/1x-screen-1x-pattern/expected.png b/metrics/expectations/platform-android/render-tests/sprites/1x-screen-1x-pattern/expected.png similarity index 100% rename from render-test/expected/android/render-tests/sprites/1x-screen-1x-pattern/expected.png rename to metrics/expectations/platform-android/render-tests/sprites/1x-screen-1x-pattern/expected.png diff --git a/render-test/expected/android/render-tests/sprites/1x-screen-2x-pattern/expected.png b/metrics/expectations/platform-android/render-tests/sprites/1x-screen-2x-pattern/expected.png similarity index 100% rename from render-test/expected/android/render-tests/sprites/1x-screen-2x-pattern/expected.png rename to metrics/expectations/platform-android/render-tests/sprites/1x-screen-2x-pattern/expected.png diff --git a/render-test/expected/android/render-tests/sprites/2x-screen-1x-pattern/expected.png b/metrics/expectations/platform-android/render-tests/sprites/2x-screen-1x-pattern/expected.png similarity index 100% rename from render-test/expected/android/render-tests/sprites/2x-screen-1x-pattern/expected.png rename to metrics/expectations/platform-android/render-tests/sprites/2x-screen-1x-pattern/expected.png diff --git a/render-test/expected/android/render-tests/sprites/2x-screen-2x-pattern/expected.png b/metrics/expectations/platform-android/render-tests/sprites/2x-screen-2x-pattern/expected.png similarity index 100% rename from render-test/expected/android/render-tests/sprites/2x-screen-2x-pattern/expected.png rename to metrics/expectations/platform-android/render-tests/sprites/2x-screen-2x-pattern/expected.png diff --git a/metrics/ignores/platform-all.json b/metrics/ignores/platform-all.json new file mode 100644 index 000000000..e1cf323af --- /dev/null +++ b/metrics/ignores/platform-all.json @@ -0,0 +1,133 @@ +{ + "probes/memory/pass-memory-size-is-same": "TODO: Check with Mikhail why is this failing", + "binary-size/android-arm64-v8a": "skip - only makes sense for the binary size bot.", + "binary-size/android-armeabi-v7a": "skip - only makes sense for the binary size bot.", + "binary-size/android-x86": "skip - only makes sense for the binary size bot.", + "binary-size/android-x86_64": "skip - only makes sense for the binary size bot.", + "binary-size/linux-clang8": "skip - only makes sense for the binary size bot.", + "binary-size/linux-gcc8": "skip - only makes sense for the binary size bot.", + "binary-size/macos-xcode11": "skip - only makes sense for the binary size bot.", + "expression-tests/collator/accent-equals-de": "Locale-specific behavior changes based on platform.", + "expression-tests/in/assert-array": "https://github.com/mapbox/mapbox-gl-native/issues/15893", + "expression-tests/in/assert-string": "https://github.com/mapbox/mapbox-gl-native/issues/15893", + "expression-tests/in/basic-array": "https://github.com/mapbox/mapbox-gl-native/issues/15893", + "expression-tests/in/basic-string": "https://github.com/mapbox/mapbox-gl-native/issues/15893", + "expression-tests/in/invalid-haystack": "https://github.com/mapbox/mapbox-gl-native/issues/15893", + "expression-tests/in/invalid-needle": "https://github.com/mapbox/mapbox-gl-native/issues/15893", + "expression-tests/interpolate-hcl/linear": "https://github.com/mapbox/mapbox-gl-native/issues/8720", + "expression-tests/interpolate-lab/linear": "https://github.com/mapbox/mapbox-gl-native/issues/8720", + "expression-tests/is-supported-script/default": "This tests RTL text plugin behavior specific to GL JS", + "expression-tests/legacy/categorical/array": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/categorical/boolean": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/categorical/color": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/categorical/number": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/categorical/string": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/exponential/color-hcl": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/exponential/color-lab": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/identity/color-default": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/identity/enum-default": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/interval/composite": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/interval/composite-default": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/interval/tokens-zoom": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/resolved-locale/basic": "Even the 'en' locale may not be present on some test systems.", + "probes/file-size/fail-file-doesnt-match": "Should fail, doesn't match the expectation.", + "probes/file-size/fail-file-not-found": "Should fail, file not found.", + "probes/file-size/fail-size-is-over": "Should fail, size is bigger than expected.", + "probes/file-size/fail-size-is-under": "Should fail, size is smaller than expected.", + "probes/gfx/fail-ib-mem-mismatch": "Should fail, combined byte size of index buffers doesn't match the expectation.", + "probes/gfx/fail-negative-framebuffer-count": "Should fail, number of frame buffers is higher than expected.", + "probes/gfx/fail-texture-mem-mismatch": "Should fail, combined byte size of textures doesn't match the expectation.", + "probes/gfx/fail-too-few-buffers": "Should fail, number of vertex and index buffers is smaller than expected.", + "probes/gfx/fail-too-few-textures": "Should fail, number of textures is smaller than expected.", + "probes/gfx/fail-too-many-drawcalls": "Should fail, number of draw calls higher than expected.", + "probes/gfx/fail-vb-mem-mismatch": "Should fail, combined byte size of index buffers doesn't match the expectation.", + "probes/memory/fail-memory-size-is-too-big": "Should fail, memory size is bigger than expected.", + "probes/memory/fail-memory-size-is-too-small": "Should fail, memory size is smaller than expected.", + "probes/network/fail-requests": "Should fail, number of requests higher than expected.", + "probes/network/fail-requests-transferred": "Should fail, number of requests higher than expected and amount of transferred data less than expected.", + "probes/network/fail-transferred": "Should fail, amount of transferred data higher than expected.", + "query-tests/fill-extrusion-translate/multiple-layers": "https://github.com/mapbox/mapbox-gl-native/issues/12701", + "query-tests/fill-extrusion/base-in": "https://github.com/mapbox/mapbox-gl-native/issues/13139", + "query-tests/fill-extrusion/box-in": "https://github.com/mapbox/mapbox-gl-native/issues/13139", + "query-tests/fill-extrusion/side-in": "https://github.com/mapbox/mapbox-gl-native/issues/13139", + "query-tests/fill-extrusion/sort": "https://github.com/mapbox/mapbox-gl-native/issues/13139", + "query-tests/fill-extrusion/sort-concave-inner": "https://github.com/mapbox/mapbox-gl-native/issues/13139", + "query-tests/fill-extrusion/sort-concave-outer": "https://github.com/mapbox/mapbox-gl-native/issues/13139", + "query-tests/fill-extrusion/sort-rotated": "https://github.com/mapbox/mapbox-gl-native/issues/13139", + "query-tests/fill-extrusion/top-in": "https://github.com/mapbox/mapbox-gl-native/issues/13139", + "query-tests/fill-translate/multiple-layers": "https://github.com/mapbox/mapbox-gl-native/issues/12701", + "query-tests/geometry/multilinestring": "needs investigation", + "query-tests/geometry/multipolygon": "needs investigation", + "query-tests/geometry/polygon": "needs investigation", + "query-tests/regressions/mapbox-gl-js#6555": "skip - no querySourceFeatures in mbgl-node; needs issue", + "query-tests/regressions/mapbox-gl-js#7883": "https://github.com/mapbox/mapbox-gl-native/issues/14585", + "query-tests/world-wrapping/box": "skip - needs issue", + "render-tests/background-color/colorSpace-hcl": "needs issue", + "render-tests/background-color/transition": "https://github.com/mapbox/mapbox-gl-native/issues/10619", + "render-tests/canvas/default": "skip - js specific", + "render-tests/canvas/update": "skip - js specific", + "render-tests/circle-sort-key/literal": "https://github.com/mapbox/mapbox-gl-native/issues/15008", + "render-tests/collator/default": "Some test platforms don't resolve 'en' locale", + "render-tests/collator/resolved-locale": "Some test platforms don't resolve 'en' locale", + "render-tests/custom-layer-js/depth": "skip - js specific", + "render-tests/custom-layer-js/null-island": "skip - js specific", + "render-tests/custom-layer-js/tent-3d": "skip - js specific", + "render-tests/debug/collision": "https://github.com/mapbox/mapbox-gl-native/issues/3841", + "render-tests/debug/overdraw": "https://github.com/mapbox/mapbox-gl-native/issues/15638", + "render-tests/debug/raster": "https://github.com/mapbox/mapbox-gl-native/issues/15510", + "render-tests/debug/tile": "https://github.com/mapbox/mapbox-gl-native/issues/3841", + "render-tests/debug/tile-overscaled": "https://github.com/mapbox/mapbox-gl-native/issues/3841", + "render-tests/extent/1024-circle": "needs investigation", + "render-tests/feature-state/promote-id": "https://github.com/mapbox/mapbox-gl-native/issues/16016", + "render-tests/fill-extrusion-geometry/linestring": "https://github.com/mapbox/mapbox-gl-native/pull/14240", + "render-tests/fill-extrusion-pattern/@2x": "https://github.com/mapbox/mapbox-gl-js/issues/3327", + "render-tests/fill-extrusion-pattern/feature-expression": "https://github.com/mapbox/mapbox-gl-js/issues/3327", + "render-tests/fill-extrusion-pattern/function": "https://github.com/mapbox/mapbox-gl-js/issues/3327", + "render-tests/fill-extrusion-pattern/function-2": "https://github.com/mapbox/mapbox-gl-js/issues/3327", + "render-tests/fill-extrusion-pattern/literal": "https://github.com/mapbox/mapbox-gl-js/issues/3327", + "render-tests/fill-extrusion-pattern/opacity": "https://github.com/mapbox/mapbox-gl-js/issues/3327", + "render-tests/fill-extrusion-pattern/tile-buffer": "https://github.com/mapbox/mapbox-gl-js/issues/3327", + "render-tests/fill-pattern/update-feature-state": "https://github.com/mapbox/mapbox-gl-native/issues/15895", + "render-tests/fill-sort-key/literal": "https://github.com/mapbox/mapbox-gl-native/issues/15008", + "render-tests/geojson/inline-linestring-fill": "current behavior is arbitrary", + "render-tests/icon-text-fit/stretch-fifteen-part": "https://github.com/mapbox/mapbox-gl-native/issues/16017", + "render-tests/icon-text-fit/stretch-nine-part": "https://github.com/mapbox/mapbox-gl-native/issues/16017", + "render-tests/icon-text-fit/stretch-nine-part-@2x": "https://github.com/mapbox/mapbox-gl-native/issues/16017", + "render-tests/icon-text-fit/stretch-nine-part-content": "https://github.com/mapbox/mapbox-gl-native/issues/16017", + "render-tests/icon-text-fit/stretch-nine-part-content-collision": "https://github.com/mapbox/mapbox-gl-native/issues/16017", + "render-tests/icon-text-fit/stretch-nine-part-just-height": "https://github.com/mapbox/mapbox-gl-native/issues/16017", + "render-tests/icon-text-fit/stretch-nine-part-just-width": "https://github.com/mapbox/mapbox-gl-native/issues/16017", + "render-tests/icon-text-fit/stretch-three-part": "https://github.com/mapbox/mapbox-gl-native/issues/16017", + "render-tests/icon-text-fit/stretch-two-part": "https://github.com/mapbox/mapbox-gl-native/issues/16017", + "render-tests/icon-text-fit/stretch-underscale": "https://github.com/mapbox/mapbox-gl-native/issues/16017", + "render-tests/icon-text-fit/text-variable-anchor-overlap": "https://github.com/mapbox/mapbox-gl-native/issues/15809", + "render-tests/line-sort-key/literal": "https://github.com/mapbox/mapbox-gl-native/issues/15008", + "render-tests/mixed-zoom/z10-z11": "https://github.com/mapbox/mapbox-gl-native/issues/10397", + "render-tests/raster-masking/overlapping-zoom": "https://github.com/mapbox/mapbox-gl-native/issues/10195", + "render-tests/real-world/bangkok": "https://github.com/mapbox/mapbox-gl-native/issues/10412", + "render-tests/real-world/chicago": "https://github.com/mapbox/mapbox-gl-native/issues/10412", + "render-tests/real-world/sanfrancisco": "https://github.com/mapbox/mapbox-gl-native/issues/10412", + "render-tests/regressions/mapbox-gl-js#2467": "https://github.com/mapbox/mapbox-gl-native/issues/10619", + "render-tests/regressions/mapbox-gl-js#2762": "https://github.com/mapbox/mapbox-gl-native/issues/10619", + "render-tests/regressions/mapbox-gl-js#2769": "https://github.com/mapbox/mapbox-gl-native/issues/10619", + "render-tests/regressions/mapbox-gl-js#5740": "https://github.com/mapbox/mapbox-gl-native/issues/10619", + "render-tests/regressions/mapbox-gl-js#5982": "https://github.com/mapbox/mapbox-gl-native/issues/10619", + "render-tests/regressions/mapbox-gl-js#6655": "skip - port https://github.com/mapbox/mapbox-gl-js/pull/6263 - needs issue", + "render-tests/regressions/mapbox-gl-js#6706": "https://github.com/mapbox/mapbox-gl-native/issues/10619", + "render-tests/regressions/mapbox-gl-js#6806": "pending https://github.com/mapbox/mapbox-gl-js/pull/6812", + "render-tests/regressions/mapbox-gl-js#7271": "https://github.com/mapbox/mapbox-gl-native/issues/12888", + "render-tests/regressions/mapbox-gl-js#7302": "skip - js specific", + "render-tests/regressions/mapbox-gl-js#7708": "skip - js specific", + "render-tests/regressions/mapbox-gl-js#8026": "skip - js specific", + "render-tests/regressions/mapbox-gl-js#8817": "skip - https://github.com/mapbox/mapbox-gl-native/issues/15737", + "render-tests/regressions/mapbox-gl-js#9009": "https://github.com/mapbox/mapbox-gl-native/issues/16018", + "render-tests/regressions/mapbox-gl-native#7357": "https://github.com/mapbox/mapbox-gl-native/issues/7357", + "render-tests/runtime-styling/image-add-remove-add": "skip - https://github.com/mapbox/mapbox-gl-native/issues/16021", + "render-tests/runtime-styling/pattern-add-remove-add": "skip - https://github.com/mapbox/mapbox-gl-native/issues/16021", + "render-tests/symbol-cross-fade/chinese": "https://github.com/mapbox/mapbox-gl-native/issues/10619", + "render-tests/text-max-width/zero-width-point-placement": "https://github.com/mapbox/mapbox-gl-native/issues/15648", + "render-tests/text-size/nan": "https://github.com/mapbox/mapbox-gl-native/issues/16020", + "render-tests/text-variable-anchor/remember-last-placement": "skip - fails on gl-native, as symbol index is not functional at static map mode - needs issue", + "render-tests/video/default": "skip - https://github.com/mapbox/mapbox-gl-native/issues/601", + "render-tests/zoomed-fill/negative-zoom": "https://github.com/mapbox/mapbox-gl-native/issues/16019" +} diff --git a/metrics/ignores/platform-android.json b/metrics/ignores/platform-android.json new file mode 100644 index 000000000..ffd8fc9e1 --- /dev/null +++ b/metrics/ignores/platform-android.json @@ -0,0 +1,13 @@ +{ + "probes/gfx/pass": "Needs to be investigated and fixed.", + "probes/gfx/pass-double-probe": "Needs to be investigated and fixed.", + "probes/gfx/pass-probe-reset": "Needs to be investigated and fixed.", + "probes/network/pass": "Needs to be investigated and fixed.", + "render-tests/icon-text-fit/both-text-anchor-1x-image-2x-screen": "Temporarily fails as android test app does not derefence symbolinks in sprites folder.", + "render-tests/icon-text-fit/both-text-anchor-2x-image-1x-screen": "Temporarily fails as android test app does not derefence symbolinks in sprites folder.", + "render-tests/line-pattern/overscaled": "Needs to be investigated and fixed.", + "render-tests/regressions/mapbox-gl-js#5911a": "Needs to be investigated and fixed.", + "render-tests/regressions/mapbox-gl-native#9976": "Needs to be investigated and fixed.", + "render-tests/runtime-styling/image-add-pattern": "Needs to be investigated and fixed.", + "render-tests/runtime-styling/image-update-pattern": "Needs to be investigated and fixed." +} diff --git a/metrics/ignores/platform-linux.json b/metrics/ignores/platform-linux.json new file mode 100644 index 000000000..0988aff5a --- /dev/null +++ b/metrics/ignores/platform-linux.json @@ -0,0 +1,10 @@ +{ + "render-tests/fill-opacity/zoom-and-property-function-pattern": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/15322", + "render-tests/fill-pattern/literal": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/14423", + "render-tests/fill-pattern/opacity": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/14870", + "render-tests/fill-pattern/zoomed": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/14768", + "render-tests/line-pattern/opacity": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/15320", + "render-tests/line-translate/literal": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/14859", + "render-tests/regressions/mapbox-gl-js#5642": "Failing with mbgl-render-test", + "render-tests/regressions/mapbox-gl-js#7066": "Failing with mbgl-render-test" +} diff --git a/metrics/ignores/platform-macos.json b/metrics/ignores/platform-macos.json new file mode 100644 index 000000000..bc5e67820 --- /dev/null +++ b/metrics/ignores/platform-macos.json @@ -0,0 +1,4 @@ +{ + "render-tests/regressions/mapbox-gl-js#5642": "Failing with mbgl-render-test", + "render-tests/regressions/mapbox-gl-js#7066": "Failing with mbgl-render-test" +} diff --git a/metrics/next-FIXME-linux-asan-style.json b/metrics/next-FIXME-linux-asan-style.json new file mode 100644 index 000000000..518337de9 --- /dev/null +++ b/metrics/next-FIXME-linux-asan-style.json @@ -0,0 +1,12 @@ +{ + "asset_path": "../mapbox-gl-js/test/integration", + "base_test_path": "../mapbox-gl-js/test/integration", + "expectation_paths": [ + "expectations/platform-all" + ], + "ignore_paths": [ + "ignores/platform-all.json", + "ignores/platform-linux.json" + ], + "vendor_path": "../vendor" +} diff --git a/metrics/next-FIXME-linux-ubsan-style.json b/metrics/next-FIXME-linux-ubsan-style.json new file mode 100644 index 000000000..518337de9 --- /dev/null +++ b/metrics/next-FIXME-linux-ubsan-style.json @@ -0,0 +1,12 @@ +{ + "asset_path": "../mapbox-gl-js/test/integration", + "base_test_path": "../mapbox-gl-js/test/integration", + "expectation_paths": [ + "expectations/platform-all" + ], + "ignore_paths": [ + "ignores/platform-all.json", + "ignores/platform-linux.json" + ], + "vendor_path": "../vendor" +} diff --git a/metrics/next-android-render-test-runner-metrics.json b/metrics/next-android-render-test-runner-metrics.json new file mode 100644 index 000000000..f60e6a50f --- /dev/null +++ b/metrics/next-android-render-test-runner-metrics.json @@ -0,0 +1,19 @@ +{ + "base_test_path": "tests", + "expectation_paths": [ + "expectations/platform-all", + "expectations/platform-android" + ], + "ignore_paths": [ + "ignores/platform-all.json", + "ignores/platform-linux.json", + "ignores/platform-android.json" + ], + "metric_path": "next-android-render-test-runner", + "probes": [ + "probeGFX", + "probeMemory", + "probeNetwork" + ], + "result_path": "/sdcard" +} diff --git a/metrics/next-android-render-test-runner-probe-gfx-network.json b/metrics/next-android-render-test-runner-probe-gfx-network.json deleted file mode 100644 index db341e43c..000000000 --- a/metrics/next-android-render-test-runner-probe-gfx-network.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "base_test_path":"mapbox-gl-js/test/integration", - "expectation_paths":["render-test/expected", "render-test/expected/android"], - "ignore_paths":["render-test/ignores/ignores.json", "render-test/ignores/linux-ignores.json", "render-test/ignores/android-temp-fail.json"], - "vendor_path":"vendor", - "asset_path": "mapbox-gl-js/test/integration", - "result_path":"/sdcard/", - "probes": ["probeGFX", "probeNetwork"], - "metric_path":"metrics/next-android-render-test-runner" -} \ No newline at end of file diff --git a/metrics/next-android-render-test-runner-probe-memory.json b/metrics/next-android-render-test-runner-probe-memory.json deleted file mode 100644 index aa99c5a61..000000000 --- a/metrics/next-android-render-test-runner-probe-memory.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "base_test_path":"mapbox-gl-js/test/integration", - "expectation_paths":["render-test/expected", "render-test/expected/android"], - "ignore_paths":["render-test/ignores/ignores.json", "render-test/ignores/linux-ignores.json", "render-test/ignores/android-temp-fail.json"], - "vendor_path":"vendor", - "asset_path": "mapbox-gl-js/test/integration", - "result_path":"/sdcard/", - "probes": ["probeMemory"], - "metric_path":"metrics/next-android-render-test-runner", - "filter": "real-world" -} \ No newline at end of file diff --git a/metrics/next-android-render-test-runner-style.json b/metrics/next-android-render-test-runner-style.json new file mode 100644 index 000000000..b00e9f0ea --- /dev/null +++ b/metrics/next-android-render-test-runner-style.json @@ -0,0 +1,20 @@ +{ + "asset_path": "../mapbox-gl-js/test/integration", + "base_test_path": "../mapbox-gl-js/test/integration", + "expectation_paths": [ + "expectations/platform-all", + "expectations/platform-android" + ], + "ignore_paths": [ + "ignores/platform-all.json", + "ignores/platform-linux.json", + "ignores/platform-android.json" + ], + "metric_path": "next-android-render-test-runner", + "probes": [ + "probeGFX", + "probeNetwork" + ], + "result_path": "/sdcard", + "vendor_path": "../vendor" +} diff --git a/metrics/next-linux-clang8-release-style.json b/metrics/next-linux-clang8-release-style.json new file mode 100644 index 000000000..518337de9 --- /dev/null +++ b/metrics/next-linux-clang8-release-style.json @@ -0,0 +1,12 @@ +{ + "asset_path": "../mapbox-gl-js/test/integration", + "base_test_path": "../mapbox-gl-js/test/integration", + "expectation_paths": [ + "expectations/platform-all" + ], + "ignore_paths": [ + "ignores/platform-all.json", + "ignores/platform-linux.json" + ], + "vendor_path": "../vendor" +} diff --git a/metrics/next-linux-gcc8-debug-coverage-style.json b/metrics/next-linux-gcc8-debug-coverage-style.json new file mode 100644 index 000000000..518337de9 --- /dev/null +++ b/metrics/next-linux-gcc8-debug-coverage-style.json @@ -0,0 +1,12 @@ +{ + "asset_path": "../mapbox-gl-js/test/integration", + "base_test_path": "../mapbox-gl-js/test/integration", + "expectation_paths": [ + "expectations/platform-all" + ], + "ignore_paths": [ + "ignores/platform-all.json", + "ignores/platform-linux.json" + ], + "vendor_path": "../vendor" +} diff --git a/metrics/next-linux-gcc8-release-metrics.json b/metrics/next-linux-gcc8-release-metrics.json new file mode 100644 index 000000000..6c041af80 --- /dev/null +++ b/metrics/next-linux-gcc8-release-metrics.json @@ -0,0 +1,18 @@ +{ + "asset_path": "../mapbox-gl-js/test/integration", + "base_test_path": "tests", + "expectation_paths": [ + "expectations/platform-all" + ], + "ignore_paths": [ + "ignores/platform-all.json", + "ignores/platform-linux.json" + ], + "metric_path": "next-linux-gcc8-release", + "probes": [ + "probeGFX", + "probeMemory", + "probeNetwork" + ], + "vendor_path": "../vendor" +} diff --git a/metrics/next-linux-gcc8-release-probe-gfx-network.json b/metrics/next-linux-gcc8-release-probe-gfx-network.json deleted file mode 100644 index 2c5f9b57a..000000000 --- a/metrics/next-linux-gcc8-release-probe-gfx-network.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "base_test_path":"../mapbox-gl-js/test/integration", - "expectation_paths":["../render-test/expected/"], - "ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"], - "vendor_path":"../vendor", - "asset_path": "../mapbox-gl-js/test/integration", - "probes": ["probeGFX", "probeNetwork"], - "metric_path":"../metrics/next-linux-gcc8-release" -} diff --git a/metrics/next-linux-gcc8-release-probe-memory.json b/metrics/next-linux-gcc8-release-probe-memory.json deleted file mode 100644 index cae354129..000000000 --- a/metrics/next-linux-gcc8-release-probe-memory.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "base_test_path":"../mapbox-gl-js/test/integration", - "expectation_paths":["../render-test/expected/"], - "ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"], - "vendor_path":"../vendor", - "asset_path": "../mapbox-gl-js/test/integration", - "probes": ["probeMemory"], - "metric_path":"../metrics/next-linux-gcc8-release", - "filter": "real-world" -} diff --git a/metrics/next-linux-gcc8-release-style.json b/metrics/next-linux-gcc8-release-style.json new file mode 100644 index 000000000..4961756a6 --- /dev/null +++ b/metrics/next-linux-gcc8-release-style.json @@ -0,0 +1,17 @@ +{ + "asset_path": "../mapbox-gl-js/test/integration", + "base_test_path": "../mapbox-gl-js/test/integration", + "expectation_paths": [ + "expectations/platform-all" + ], + "ignore_paths": [ + "ignores/platform-all.json", + "ignores/platform-linux.json" + ], + "metric_path": "next-linux-gcc8-release", + "probes": [ + "probeGFX", + "probeNetwork" + ], + "vendor_path": "../vendor" +} diff --git a/metrics/next-linux-tsan-style.json b/metrics/next-linux-tsan-style.json new file mode 100644 index 000000000..518337de9 --- /dev/null +++ b/metrics/next-linux-tsan-style.json @@ -0,0 +1,12 @@ +{ + "asset_path": "../mapbox-gl-js/test/integration", + "base_test_path": "../mapbox-gl-js/test/integration", + "expectation_paths": [ + "expectations/platform-all" + ], + "ignore_paths": [ + "ignores/platform-all.json", + "ignores/platform-linux.json" + ], + "vendor_path": "../vendor" +} diff --git a/metrics/next-macos-xcode11-debug-style.json b/metrics/next-macos-xcode11-debug-style.json new file mode 100644 index 000000000..463e771a1 --- /dev/null +++ b/metrics/next-macos-xcode11-debug-style.json @@ -0,0 +1,12 @@ +{ + "asset_path": "../mapbox-gl-js/test/integration", + "base_test_path": "../mapbox-gl-js/test/integration", + "expectation_paths": [ + "expectations/platform-all" + ], + "ignore_paths": [ + "ignores/platform-all.json", + "ignores/platform-macos.json" + ], + "vendor_path": "../vendor" +} diff --git a/metrics/next-macos-xcode11-release-style.json b/metrics/next-macos-xcode11-release-style.json new file mode 100644 index 000000000..463e771a1 --- /dev/null +++ b/metrics/next-macos-xcode11-release-style.json @@ -0,0 +1,12 @@ +{ + "asset_path": "../mapbox-gl-js/test/integration", + "base_test_path": "../mapbox-gl-js/test/integration", + "expectation_paths": [ + "expectations/platform-all" + ], + "ignore_paths": [ + "ignores/platform-all.json", + "ignores/platform-macos.json" + ], + "vendor_path": "../vendor" +} diff --git a/render-test/tests/file-size/fail-file-doesnt-match/expected.png b/metrics/tests/probes/file-size/fail-file-doesnt-match/expected.png similarity index 100% rename from render-test/tests/file-size/fail-file-doesnt-match/expected.png rename to metrics/tests/probes/file-size/fail-file-doesnt-match/expected.png diff --git a/render-test/tests/file-size/fail-file-doesnt-match/metrics.json b/metrics/tests/probes/file-size/fail-file-doesnt-match/metrics.json similarity index 100% rename from render-test/tests/file-size/fail-file-doesnt-match/metrics.json rename to metrics/tests/probes/file-size/fail-file-doesnt-match/metrics.json diff --git a/render-test/tests/file-size/fail-file-doesnt-match/style.json b/metrics/tests/probes/file-size/fail-file-doesnt-match/style.json similarity index 100% rename from render-test/tests/file-size/fail-file-doesnt-match/style.json rename to metrics/tests/probes/file-size/fail-file-doesnt-match/style.json diff --git a/render-test/tests/file-size/fail-file-not-found/expected.png b/metrics/tests/probes/file-size/fail-file-not-found/expected.png similarity index 100% rename from render-test/tests/file-size/fail-file-not-found/expected.png rename to metrics/tests/probes/file-size/fail-file-not-found/expected.png diff --git a/render-test/tests/file-size/fail-file-not-found/style.json b/metrics/tests/probes/file-size/fail-file-not-found/style.json similarity index 100% rename from render-test/tests/file-size/fail-file-not-found/style.json rename to metrics/tests/probes/file-size/fail-file-not-found/style.json diff --git a/render-test/tests/file-size/fail-size-is-over/expected.png b/metrics/tests/probes/file-size/fail-size-is-over/expected.png similarity index 100% rename from render-test/tests/file-size/fail-size-is-over/expected.png rename to metrics/tests/probes/file-size/fail-size-is-over/expected.png diff --git a/render-test/tests/file-size/fail-size-is-over/metrics.json b/metrics/tests/probes/file-size/fail-size-is-over/metrics.json similarity index 100% rename from render-test/tests/file-size/fail-size-is-over/metrics.json rename to metrics/tests/probes/file-size/fail-size-is-over/metrics.json diff --git a/render-test/tests/file-size/fail-size-is-over/style.json b/metrics/tests/probes/file-size/fail-size-is-over/style.json similarity index 100% rename from render-test/tests/file-size/fail-size-is-over/style.json rename to metrics/tests/probes/file-size/fail-size-is-over/style.json diff --git a/render-test/tests/file-size/fail-size-is-under/expected.png b/metrics/tests/probes/file-size/fail-size-is-under/expected.png similarity index 100% rename from render-test/tests/file-size/fail-size-is-under/expected.png rename to metrics/tests/probes/file-size/fail-size-is-under/expected.png diff --git a/render-test/tests/file-size/fail-size-is-under/metrics.json b/metrics/tests/probes/file-size/fail-size-is-under/metrics.json similarity index 100% rename from render-test/tests/file-size/fail-size-is-under/metrics.json rename to metrics/tests/probes/file-size/fail-size-is-under/metrics.json diff --git a/render-test/tests/file-size/fail-size-is-under/style.json b/metrics/tests/probes/file-size/fail-size-is-under/style.json similarity index 100% rename from render-test/tests/file-size/fail-size-is-under/style.json rename to metrics/tests/probes/file-size/fail-size-is-under/style.json diff --git a/render-test/tests/file-size/pass-size-is-in-tolerance-higher/expected.png b/metrics/tests/probes/file-size/pass-size-is-in-tolerance-higher/expected.png similarity index 100% rename from render-test/tests/file-size/pass-size-is-in-tolerance-higher/expected.png rename to metrics/tests/probes/file-size/pass-size-is-in-tolerance-higher/expected.png diff --git a/render-test/tests/file-size/pass-size-is-in-tolerance-higher/metrics.json b/metrics/tests/probes/file-size/pass-size-is-in-tolerance-higher/metrics.json similarity index 100% rename from render-test/tests/file-size/pass-size-is-in-tolerance-higher/metrics.json rename to metrics/tests/probes/file-size/pass-size-is-in-tolerance-higher/metrics.json diff --git a/render-test/tests/file-size/pass-size-is-in-tolerance-higher/style.json b/metrics/tests/probes/file-size/pass-size-is-in-tolerance-higher/style.json similarity index 100% rename from render-test/tests/file-size/pass-size-is-in-tolerance-higher/style.json rename to metrics/tests/probes/file-size/pass-size-is-in-tolerance-higher/style.json diff --git a/render-test/tests/file-size/pass-size-is-in-tolerance-lower/expected.png b/metrics/tests/probes/file-size/pass-size-is-in-tolerance-lower/expected.png similarity index 100% rename from render-test/tests/file-size/pass-size-is-in-tolerance-lower/expected.png rename to metrics/tests/probes/file-size/pass-size-is-in-tolerance-lower/expected.png diff --git a/render-test/tests/file-size/pass-size-is-in-tolerance-lower/metrics.json b/metrics/tests/probes/file-size/pass-size-is-in-tolerance-lower/metrics.json similarity index 100% rename from render-test/tests/file-size/pass-size-is-in-tolerance-lower/metrics.json rename to metrics/tests/probes/file-size/pass-size-is-in-tolerance-lower/metrics.json diff --git a/render-test/tests/file-size/pass-size-is-in-tolerance-lower/style.json b/metrics/tests/probes/file-size/pass-size-is-in-tolerance-lower/style.json similarity index 100% rename from render-test/tests/file-size/pass-size-is-in-tolerance-lower/style.json rename to metrics/tests/probes/file-size/pass-size-is-in-tolerance-lower/style.json diff --git a/render-test/tests/file-size/pass-size-is-same/expected.png b/metrics/tests/probes/file-size/pass-size-is-same/expected.png similarity index 100% rename from render-test/tests/file-size/pass-size-is-same/expected.png rename to metrics/tests/probes/file-size/pass-size-is-same/expected.png diff --git a/render-test/tests/file-size/pass-size-is-same/metrics.json b/metrics/tests/probes/file-size/pass-size-is-same/metrics.json similarity index 100% rename from render-test/tests/file-size/pass-size-is-same/metrics.json rename to metrics/tests/probes/file-size/pass-size-is-same/metrics.json diff --git a/render-test/tests/file-size/pass-size-is-same/style.json b/metrics/tests/probes/file-size/pass-size-is-same/style.json similarity index 100% rename from render-test/tests/file-size/pass-size-is-same/style.json rename to metrics/tests/probes/file-size/pass-size-is-same/style.json diff --git a/render-test/tests/gfx/fail-ib-mem-mismatch/expected.png b/metrics/tests/probes/gfx/fail-ib-mem-mismatch/expected.png similarity index 100% rename from render-test/tests/gfx/fail-ib-mem-mismatch/expected.png rename to metrics/tests/probes/gfx/fail-ib-mem-mismatch/expected.png diff --git a/render-test/tests/gfx/fail-ib-mem-mismatch/metrics.json b/metrics/tests/probes/gfx/fail-ib-mem-mismatch/metrics.json similarity index 100% rename from render-test/tests/gfx/fail-ib-mem-mismatch/metrics.json rename to metrics/tests/probes/gfx/fail-ib-mem-mismatch/metrics.json diff --git a/render-test/tests/gfx/fail-ib-mem-mismatch/style.json b/metrics/tests/probes/gfx/fail-ib-mem-mismatch/style.json similarity index 100% rename from render-test/tests/gfx/fail-ib-mem-mismatch/style.json rename to metrics/tests/probes/gfx/fail-ib-mem-mismatch/style.json diff --git a/render-test/tests/gfx/fail-negative-framebuffer-count/expected.png b/metrics/tests/probes/gfx/fail-negative-framebuffer-count/expected.png similarity index 100% rename from render-test/tests/gfx/fail-negative-framebuffer-count/expected.png rename to metrics/tests/probes/gfx/fail-negative-framebuffer-count/expected.png diff --git a/render-test/tests/gfx/fail-negative-framebuffer-count/metrics.json b/metrics/tests/probes/gfx/fail-negative-framebuffer-count/metrics.json similarity index 100% rename from render-test/tests/gfx/fail-negative-framebuffer-count/metrics.json rename to metrics/tests/probes/gfx/fail-negative-framebuffer-count/metrics.json diff --git a/render-test/tests/gfx/fail-negative-framebuffer-count/style.json b/metrics/tests/probes/gfx/fail-negative-framebuffer-count/style.json similarity index 100% rename from render-test/tests/gfx/fail-negative-framebuffer-count/style.json rename to metrics/tests/probes/gfx/fail-negative-framebuffer-count/style.json diff --git a/render-test/tests/gfx/fail-texture-mem-mismatch/expected.png b/metrics/tests/probes/gfx/fail-texture-mem-mismatch/expected.png similarity index 100% rename from render-test/tests/gfx/fail-texture-mem-mismatch/expected.png rename to metrics/tests/probes/gfx/fail-texture-mem-mismatch/expected.png diff --git a/render-test/tests/gfx/fail-texture-mem-mismatch/metrics.json b/metrics/tests/probes/gfx/fail-texture-mem-mismatch/metrics.json similarity index 100% rename from render-test/tests/gfx/fail-texture-mem-mismatch/metrics.json rename to metrics/tests/probes/gfx/fail-texture-mem-mismatch/metrics.json diff --git a/render-test/tests/gfx/fail-texture-mem-mismatch/style.json b/metrics/tests/probes/gfx/fail-texture-mem-mismatch/style.json similarity index 100% rename from render-test/tests/gfx/fail-texture-mem-mismatch/style.json rename to metrics/tests/probes/gfx/fail-texture-mem-mismatch/style.json diff --git a/render-test/tests/gfx/fail-too-few-buffers/expected.png b/metrics/tests/probes/gfx/fail-too-few-buffers/expected.png similarity index 100% rename from render-test/tests/gfx/fail-too-few-buffers/expected.png rename to metrics/tests/probes/gfx/fail-too-few-buffers/expected.png diff --git a/render-test/tests/gfx/fail-too-few-buffers/metrics.json b/metrics/tests/probes/gfx/fail-too-few-buffers/metrics.json similarity index 100% rename from render-test/tests/gfx/fail-too-few-buffers/metrics.json rename to metrics/tests/probes/gfx/fail-too-few-buffers/metrics.json diff --git a/render-test/tests/gfx/fail-too-few-buffers/style.json b/metrics/tests/probes/gfx/fail-too-few-buffers/style.json similarity index 100% rename from render-test/tests/gfx/fail-too-few-buffers/style.json rename to metrics/tests/probes/gfx/fail-too-few-buffers/style.json diff --git a/render-test/tests/gfx/fail-too-few-textures/expected.png b/metrics/tests/probes/gfx/fail-too-few-textures/expected.png similarity index 100% rename from render-test/tests/gfx/fail-too-few-textures/expected.png rename to metrics/tests/probes/gfx/fail-too-few-textures/expected.png diff --git a/render-test/tests/gfx/fail-too-few-textures/metrics.json b/metrics/tests/probes/gfx/fail-too-few-textures/metrics.json similarity index 100% rename from render-test/tests/gfx/fail-too-few-textures/metrics.json rename to metrics/tests/probes/gfx/fail-too-few-textures/metrics.json diff --git a/render-test/tests/gfx/fail-too-few-textures/style.json b/metrics/tests/probes/gfx/fail-too-few-textures/style.json similarity index 100% rename from render-test/tests/gfx/fail-too-few-textures/style.json rename to metrics/tests/probes/gfx/fail-too-few-textures/style.json diff --git a/render-test/tests/gfx/fail-too-many-drawcalls/expected.png b/metrics/tests/probes/gfx/fail-too-many-drawcalls/expected.png similarity index 100% rename from render-test/tests/gfx/fail-too-many-drawcalls/expected.png rename to metrics/tests/probes/gfx/fail-too-many-drawcalls/expected.png diff --git a/render-test/tests/gfx/fail-too-many-drawcalls/metrics.json b/metrics/tests/probes/gfx/fail-too-many-drawcalls/metrics.json similarity index 100% rename from render-test/tests/gfx/fail-too-many-drawcalls/metrics.json rename to metrics/tests/probes/gfx/fail-too-many-drawcalls/metrics.json diff --git a/render-test/tests/gfx/fail-too-many-drawcalls/style.json b/metrics/tests/probes/gfx/fail-too-many-drawcalls/style.json similarity index 100% rename from render-test/tests/gfx/fail-too-many-drawcalls/style.json rename to metrics/tests/probes/gfx/fail-too-many-drawcalls/style.json diff --git a/render-test/tests/gfx/fail-vb-mem-mismatch/expected.png b/metrics/tests/probes/gfx/fail-vb-mem-mismatch/expected.png similarity index 100% rename from render-test/tests/gfx/fail-vb-mem-mismatch/expected.png rename to metrics/tests/probes/gfx/fail-vb-mem-mismatch/expected.png diff --git a/render-test/tests/gfx/fail-vb-mem-mismatch/metrics.json b/metrics/tests/probes/gfx/fail-vb-mem-mismatch/metrics.json similarity index 100% rename from render-test/tests/gfx/fail-vb-mem-mismatch/metrics.json rename to metrics/tests/probes/gfx/fail-vb-mem-mismatch/metrics.json diff --git a/render-test/tests/gfx/fail-vb-mem-mismatch/style.json b/metrics/tests/probes/gfx/fail-vb-mem-mismatch/style.json similarity index 100% rename from render-test/tests/gfx/fail-vb-mem-mismatch/style.json rename to metrics/tests/probes/gfx/fail-vb-mem-mismatch/style.json diff --git a/render-test/tests/gfx/pass-double-probe/expected.png b/metrics/tests/probes/gfx/pass-double-probe/expected.png similarity index 100% rename from render-test/tests/gfx/pass-double-probe/expected.png rename to metrics/tests/probes/gfx/pass-double-probe/expected.png diff --git a/render-test/tests/gfx/pass-double-probe/metrics.json b/metrics/tests/probes/gfx/pass-double-probe/metrics.json similarity index 100% rename from render-test/tests/gfx/pass-double-probe/metrics.json rename to metrics/tests/probes/gfx/pass-double-probe/metrics.json diff --git a/render-test/tests/gfx/pass-double-probe/style.json b/metrics/tests/probes/gfx/pass-double-probe/style.json similarity index 100% rename from render-test/tests/gfx/pass-double-probe/style.json rename to metrics/tests/probes/gfx/pass-double-probe/style.json diff --git a/render-test/tests/gfx/pass-probe-reset/expected.png b/metrics/tests/probes/gfx/pass-probe-reset/expected.png similarity index 100% rename from render-test/tests/gfx/pass-probe-reset/expected.png rename to metrics/tests/probes/gfx/pass-probe-reset/expected.png diff --git a/render-test/tests/gfx/pass-probe-reset/metrics.json b/metrics/tests/probes/gfx/pass-probe-reset/metrics.json similarity index 100% rename from render-test/tests/gfx/pass-probe-reset/metrics.json rename to metrics/tests/probes/gfx/pass-probe-reset/metrics.json diff --git a/render-test/tests/gfx/pass-probe-reset/style.json b/metrics/tests/probes/gfx/pass-probe-reset/style.json similarity index 100% rename from render-test/tests/gfx/pass-probe-reset/style.json rename to metrics/tests/probes/gfx/pass-probe-reset/style.json diff --git a/render-test/tests/gfx/pass/expected.png b/metrics/tests/probes/gfx/pass/expected.png similarity index 100% rename from render-test/tests/gfx/pass/expected.png rename to metrics/tests/probes/gfx/pass/expected.png diff --git a/render-test/tests/gfx/pass/metrics.json b/metrics/tests/probes/gfx/pass/metrics.json similarity index 100% rename from render-test/tests/gfx/pass/metrics.json rename to metrics/tests/probes/gfx/pass/metrics.json diff --git a/render-test/tests/gfx/pass/style.json b/metrics/tests/probes/gfx/pass/style.json similarity index 100% rename from render-test/tests/gfx/pass/style.json rename to metrics/tests/probes/gfx/pass/style.json diff --git a/render-test/expected/tests/memory/pass-memory-size-is-same/expected.png b/metrics/tests/probes/memory/fail-memory-size-is-too-big/expected.png similarity index 100% rename from render-test/expected/tests/memory/pass-memory-size-is-same/expected.png rename to metrics/tests/probes/memory/fail-memory-size-is-too-big/expected.png diff --git a/render-test/tests/memory/fail-memory-size-is-too-big/metrics.json b/metrics/tests/probes/memory/fail-memory-size-is-too-big/metrics.json similarity index 100% rename from render-test/tests/memory/fail-memory-size-is-too-big/metrics.json rename to metrics/tests/probes/memory/fail-memory-size-is-too-big/metrics.json diff --git a/render-test/tests/memory/fail-memory-size-is-too-big/style.json b/metrics/tests/probes/memory/fail-memory-size-is-too-big/style.json similarity index 100% rename from render-test/tests/memory/fail-memory-size-is-too-big/style.json rename to metrics/tests/probes/memory/fail-memory-size-is-too-big/style.json diff --git a/render-test/tests/memory/fail-memory-size-is-too-big/expected.png b/metrics/tests/probes/memory/fail-memory-size-is-too-small/expected.png similarity index 100% rename from render-test/tests/memory/fail-memory-size-is-too-big/expected.png rename to metrics/tests/probes/memory/fail-memory-size-is-too-small/expected.png diff --git a/render-test/tests/memory/fail-memory-size-is-too-small/metrics.json b/metrics/tests/probes/memory/fail-memory-size-is-too-small/metrics.json similarity index 100% rename from render-test/tests/memory/fail-memory-size-is-too-small/metrics.json rename to metrics/tests/probes/memory/fail-memory-size-is-too-small/metrics.json diff --git a/render-test/tests/memory/fail-memory-size-is-too-small/style.json b/metrics/tests/probes/memory/fail-memory-size-is-too-small/style.json similarity index 100% rename from render-test/tests/memory/fail-memory-size-is-too-small/style.json rename to metrics/tests/probes/memory/fail-memory-size-is-too-small/style.json diff --git a/render-test/tests/memory/fail-memory-size-is-too-small/expected.png b/metrics/tests/probes/memory/pass-memory-size-is-same/expected.png similarity index 100% rename from render-test/tests/memory/fail-memory-size-is-too-small/expected.png rename to metrics/tests/probes/memory/pass-memory-size-is-same/expected.png diff --git a/render-test/tests/memory/pass-memory-size-is-same/metrics.json b/metrics/tests/probes/memory/pass-memory-size-is-same/metrics.json similarity index 100% rename from render-test/tests/memory/pass-memory-size-is-same/metrics.json rename to metrics/tests/probes/memory/pass-memory-size-is-same/metrics.json diff --git a/render-test/tests/memory/pass-memory-size-is-same/style.json b/metrics/tests/probes/memory/pass-memory-size-is-same/style.json similarity index 100% rename from render-test/tests/memory/pass-memory-size-is-same/style.json rename to metrics/tests/probes/memory/pass-memory-size-is-same/style.json diff --git a/render-test/tests/network/fail-requests-transferred/expected.png b/metrics/tests/probes/network/fail-requests-transferred/expected.png similarity index 100% rename from render-test/tests/network/fail-requests-transferred/expected.png rename to metrics/tests/probes/network/fail-requests-transferred/expected.png diff --git a/render-test/tests/network/fail-requests-transferred/metrics.json b/metrics/tests/probes/network/fail-requests-transferred/metrics.json similarity index 100% rename from render-test/tests/network/fail-requests-transferred/metrics.json rename to metrics/tests/probes/network/fail-requests-transferred/metrics.json diff --git a/render-test/tests/network/fail-requests-transferred/style.json b/metrics/tests/probes/network/fail-requests-transferred/style.json similarity index 100% rename from render-test/tests/network/fail-requests-transferred/style.json rename to metrics/tests/probes/network/fail-requests-transferred/style.json diff --git a/render-test/tests/network/fail-requests/expected.png b/metrics/tests/probes/network/fail-requests/expected.png similarity index 100% rename from render-test/tests/network/fail-requests/expected.png rename to metrics/tests/probes/network/fail-requests/expected.png diff --git a/render-test/tests/network/fail-requests/metrics.json b/metrics/tests/probes/network/fail-requests/metrics.json similarity index 100% rename from render-test/tests/network/fail-requests/metrics.json rename to metrics/tests/probes/network/fail-requests/metrics.json diff --git a/render-test/tests/network/fail-requests/style.json b/metrics/tests/probes/network/fail-requests/style.json similarity index 100% rename from render-test/tests/network/fail-requests/style.json rename to metrics/tests/probes/network/fail-requests/style.json diff --git a/render-test/tests/network/fail-transferred/expected.png b/metrics/tests/probes/network/fail-transferred/expected.png similarity index 100% rename from render-test/tests/network/fail-transferred/expected.png rename to metrics/tests/probes/network/fail-transferred/expected.png diff --git a/render-test/tests/network/fail-transferred/metrics.json b/metrics/tests/probes/network/fail-transferred/metrics.json similarity index 100% rename from render-test/tests/network/fail-transferred/metrics.json rename to metrics/tests/probes/network/fail-transferred/metrics.json diff --git a/render-test/tests/network/fail-transferred/style.json b/metrics/tests/probes/network/fail-transferred/style.json similarity index 100% rename from render-test/tests/network/fail-transferred/style.json rename to metrics/tests/probes/network/fail-transferred/style.json diff --git a/render-test/tests/network/pass/expected.png b/metrics/tests/probes/network/pass/expected.png similarity index 100% rename from render-test/tests/network/pass/expected.png rename to metrics/tests/probes/network/pass/expected.png diff --git a/render-test/tests/network/pass/metrics.json b/metrics/tests/probes/network/pass/metrics.json similarity index 100% rename from render-test/tests/network/pass/metrics.json rename to metrics/tests/probes/network/pass/metrics.json diff --git a/render-test/tests/network/pass/style.json b/metrics/tests/probes/network/pass/style.json similarity index 100% rename from render-test/tests/network/pass/style.json rename to metrics/tests/probes/network/pass/style.json diff --git a/next/platform/android/android.cmake b/next/platform/android/android.cmake index 559baf3cb..52c932395 100644 --- a/next/platform/android/android.cmake +++ b/next/platform/android/android.cmake @@ -354,24 +354,6 @@ target_link_libraries( add_custom_command( TARGET mbgl-render-test-runner PRE_BUILD - COMMAND - ${CMAKE_COMMAND} - -E - copy - ${MBGL_ROOT}/metrics/next-android-render-test-runner-probe-memory.json - ${MBGL_ROOT}/android-manifest-probe-memory.json - COMMAND - ${CMAKE_COMMAND} - -E - copy - ${MBGL_ROOT}/metrics/next-android-render-test-runner-probe-gfx-network.json - ${MBGL_ROOT}/android-manifest-probe-network-gfx.json - COMMAND - ${CMAKE_COMMAND} - -E - copy - ${MBGL_ROOT}/platform/node/test/ignores.json - ${MBGL_ROOT}/render-test/ignores/ignores.json COMMAND ${CMAKE_COMMAND} -E @@ -380,11 +362,6 @@ add_custom_command( "render-test/android/app/src/main/assets/data.zip" --format=zip --files-from=render-test/android/app/src/main/assets/to_zip.txt - COMMAND - ${CMAKE_COMMAND} - -E - remove - ${MBGL_ROOT}/android-manifest* WORKING_DIRECTORY ${MBGL_ROOT} ) diff --git a/next/platform/linux/linux.cmake b/next/platform/linux/linux.cmake index cb866d833..c9cdd2754 100644 --- a/next/platform/linux/linux.cmake +++ b/next/platform/linux/linux.cmake @@ -121,23 +121,3 @@ target_link_libraries( add_test(NAME mbgl-benchmark-runner COMMAND mbgl-benchmark-runner WORKING_DIRECTORY ${MBGL_ROOT}) add_test(NAME mbgl-test-runner COMMAND mbgl-test-runner WORKING_DIRECTORY ${MBGL_ROOT}) - -string(RANDOM LENGTH 5 ALPHABET 0123456789 MBGL_RENDER_TEST_SEED) - -add_test( - NAME mbgl-render-test - COMMAND - mbgl-render-test-runner - render-tests - --recycle-map - --shuffle - --manifestPath=${MBGL_ROOT}/render-test/linux-manifest.json - --seed=${MBGL_RENDER_TEST_SEED} -) - -add_test( - NAME mbgl-render-test-probe-unit-tests - COMMAND mbgl-render-test-runner tests --manifestPath=${MBGL_ROOT}/render-test/linux-probe-manifest.json -) - -add_test(NAME mbgl-query-test COMMAND mbgl-render-test-runner query-tests --manifestPath=${MBGL_ROOT}/render-test/linux-query-manifest.json) diff --git a/next/platform/macos/macos.cmake b/next/platform/macos/macos.cmake index 6f4e67645..9a7f538db 100644 --- a/next/platform/macos/macos.cmake +++ b/next/platform/macos/macos.cmake @@ -200,20 +200,3 @@ set_property(TARGET mbgl-render-test-runner PROPERTY FOLDER Executables) add_test(NAME mbgl-benchmark-runner COMMAND mbgl-benchmark-runner WORKING_DIRECTORY ${MBGL_ROOT}) add_test(NAME mbgl-test-runner COMMAND mbgl-test-runner WORKING_DIRECTORY ${MBGL_ROOT}) - -string(RANDOM LENGTH 5 ALPHABET 0123456789 MBGL_RENDER_TEST_SEED) - -add_test( - NAME mbgl-render-test - COMMAND - mbgl-render-test-runner - render-tests - --recycle-map - --shuffle - --manifestPath=${MBGL_ROOT}/render-test/mac-manifest.json - --seed=${MBGL_RENDER_TEST_SEED} -) - -add_test(NAME mbgl-render-test-probes COMMAND mbgl-render-test-runner tests --manifestPath=${MBGL_ROOT}/render-test/mac-probe-manifest.json) - -add_test(NAME mbgl-query-test COMMAND mbgl-render-test-runner query-tests --manifestPath=${MBGL_ROOT}/render-test/mac-manifest.json) diff --git a/next/platform/node/CMakeLists.txt b/next/platform/node/CMakeLists.txt index adabd620c..22a975afe 100644 --- a/next/platform/node/CMakeLists.txt +++ b/next/platform/node/CMakeLists.txt @@ -48,6 +48,5 @@ target_link_libraries( # not work here because node will do a dlopen(), which is not currently supported. if(CMAKE_SYSTEM_NAME STREQUAL Linux AND NOT MBGL_WITH_SANITIZER) add_test(NAME mbgl-node-memory COMMAND npm run test-memory WORKING_DIRECTORY ${MBGL_ROOT}) - add_test(NAME mbgl-node-query COMMAND npm run test-query WORKING_DIRECTORY ${MBGL_ROOT}) add_test(NAME mbgl-node-test COMMAND npm run test WORKING_DIRECTORY ${MBGL_ROOT}) endif() diff --git a/platform/android/src/test/render_test_runner.cpp b/platform/android/src/test/render_test_runner.cpp index e738d7a8b..fe6148f95 100644 --- a/platform/android/src/test/render_test_runner.cpp +++ b/platform/android/src/test/render_test_runner.cpp @@ -283,8 +283,8 @@ void android_main(struct android_app* app) { return result; }; - auto result = runTestWithManifest("/android-manifest-probe-network-gfx.json"); - result = runTestWithManifest("/android-manifest-probe-memory.json") && result; + auto result = runTestWithManifest("/metrics/next-android-render-test-runner-metrics.json"); + result = runTestWithManifest("/metrics/next-android-render-test-runner-style.json") && result; mbgl::Log::Info(mbgl::Event::General, "All tests are finished!"); changeState(env, app, result); } diff --git a/platform/node/test/ignores.json b/platform/node/test/ignores.json deleted file mode 100644 index f086e6078..000000000 --- a/platform/node/test/ignores.json +++ /dev/null @@ -1,109 +0,0 @@ -{ - "expression-tests/collator/accent-equals-de": "Locale-specific behavior changes based on platform.", - "expression-tests/interpolate-hcl/linear": "https://github.com/mapbox/mapbox-gl-native/issues/8720", - "expression-tests/interpolate-lab/linear": "https://github.com/mapbox/mapbox-gl-native/issues/8720", - "expression-tests/is-supported-script/default": "This tests RTL text plugin behavior specific to GL JS", - "expression-tests/resolved-locale/basic": "Even the 'en' locale may not be present on some test systems.", - "expression-tests/legacy/categorical/array": "https://github.com/mapbox/mapbox-gl-native/issues/12747", - "expression-tests/legacy/categorical/boolean": "https://github.com/mapbox/mapbox-gl-native/issues/12747", - "expression-tests/legacy/categorical/color": "https://github.com/mapbox/mapbox-gl-native/issues/12747", - "expression-tests/legacy/categorical/number": "https://github.com/mapbox/mapbox-gl-native/issues/12747", - "expression-tests/legacy/categorical/string": "https://github.com/mapbox/mapbox-gl-native/issues/12747", - "expression-tests/legacy/exponential/color-hcl": "https://github.com/mapbox/mapbox-gl-native/issues/12747", - "expression-tests/legacy/exponential/color-lab": "https://github.com/mapbox/mapbox-gl-native/issues/12747", - "expression-tests/legacy/identity/color-default": "https://github.com/mapbox/mapbox-gl-native/issues/12747", - "expression-tests/legacy/identity/enum-default": "https://github.com/mapbox/mapbox-gl-native/issues/12747", - "expression-tests/legacy/interval/composite": "https://github.com/mapbox/mapbox-gl-native/issues/12747", - "expression-tests/legacy/interval/composite-default": "https://github.com/mapbox/mapbox-gl-native/issues/12747", - "expression-tests/legacy/interval/tokens-zoom": "https://github.com/mapbox/mapbox-gl-native/issues/12747", - "expression-tests/in/assert-array": "https://github.com/mapbox/mapbox-gl-native/issues/15893", - "expression-tests/in/assert-string": "https://github.com/mapbox/mapbox-gl-native/issues/15893", - "expression-tests/in/basic-array": "https://github.com/mapbox/mapbox-gl-native/issues/15893", - "expression-tests/in/basic-string": "https://github.com/mapbox/mapbox-gl-native/issues/15893", - "expression-tests/in/invalid-haystack": "https://github.com/mapbox/mapbox-gl-native/issues/15893", - "expression-tests/in/invalid-needle": "https://github.com/mapbox/mapbox-gl-native/issues/15893", - "query-tests/geometry/multilinestring": "needs investigation", - "query-tests/geometry/multipolygon": "needs investigation", - "query-tests/geometry/polygon": "needs investigation", - "query-tests/world-wrapping/box": "skip - needs issue", - "query-tests/fill-extrusion-translate/multiple-layers": "https://github.com/mapbox/mapbox-gl-native/issues/12701", - "query-tests/fill-translate/multiple-layers": "https://github.com/mapbox/mapbox-gl-native/issues/12701", - "query-tests/regressions/mapbox-gl-js#6555": "skip - no querySourceFeatures in mbgl-node; needs issue", - "render-tests/background-color/transition": "https://github.com/mapbox/mapbox-gl-native/issues/10619", - "render-tests/canvas/default": "skip - js specific", - "render-tests/canvas/update": "skip - js specific", - "render-tests/collator/resolved-locale": "Some test platforms don't resolve 'en' locale", - "render-tests/collator/default": "Some test platforms don't resolve 'en' locale", - "render-tests/custom-layer-js/depth": "skip - js specific", - "render-tests/custom-layer-js/null-island": "skip - js specific", - "render-tests/custom-layer-js/tent-3d": "skip - js specific", - "render-tests/regressions/mapbox-gl-js#7708": "skip - js specific", - "render-tests/debug/overdraw": "https://github.com/mapbox/mapbox-gl-native/issues/15638", - "render-tests/debug/collision": "https://github.com/mapbox/mapbox-gl-native/issues/3841", - "render-tests/debug/tile": "https://github.com/mapbox/mapbox-gl-native/issues/3841", - "render-tests/debug/tile-overscaled": "https://github.com/mapbox/mapbox-gl-native/issues/3841", - "render-tests/debug/raster": "https://github.com/mapbox/mapbox-gl-native/issues/15510", - "render-tests/extent/1024-circle": "needs investigation", - "render-tests/fill-extrusion-pattern/@2x": "https://github.com/mapbox/mapbox-gl-js/issues/3327", - "render-tests/fill-extrusion-pattern/function": "https://github.com/mapbox/mapbox-gl-js/issues/3327", - "render-tests/fill-extrusion-pattern/function-2": "https://github.com/mapbox/mapbox-gl-js/issues/3327", - "render-tests/fill-extrusion-pattern/literal": "https://github.com/mapbox/mapbox-gl-js/issues/3327", - "render-tests/fill-extrusion-pattern/opacity": "https://github.com/mapbox/mapbox-gl-js/issues/3327", - "render-tests/fill-extrusion-pattern/feature-expression": "https://github.com/mapbox/mapbox-gl-js/issues/3327", - "render-tests/fill-extrusion-pattern/tile-buffer": "https://github.com/mapbox/mapbox-gl-js/issues/3327", - "render-tests/geojson/inline-linestring-fill": "current behavior is arbitrary", - "render-tests/mixed-zoom/z10-z11": "https://github.com/mapbox/mapbox-gl-native/issues/10397", - "render-tests/raster-masking/overlapping-zoom": "https://github.com/mapbox/mapbox-gl-native/issues/10195", - "render-tests/real-world/bangkok": "https://github.com/mapbox/mapbox-gl-native/issues/10412", - "render-tests/real-world/chicago": "https://github.com/mapbox/mapbox-gl-native/issues/10412", - "render-tests/real-world/sanfrancisco": "https://github.com/mapbox/mapbox-gl-native/issues/10412", - "render-tests/regressions/mapbox-gl-js#2467": "https://github.com/mapbox/mapbox-gl-native/issues/10619", - "render-tests/regressions/mapbox-gl-js#2762": "https://github.com/mapbox/mapbox-gl-native/issues/10619", - "render-tests/regressions/mapbox-gl-js#2769": "https://github.com/mapbox/mapbox-gl-native/issues/10619", - "render-tests/regressions/mapbox-gl-js#5740": "https://github.com/mapbox/mapbox-gl-native/issues/10619", - "render-tests/regressions/mapbox-gl-js#5982": "https://github.com/mapbox/mapbox-gl-native/issues/10619", - "render-tests/regressions/mapbox-gl-js#6655": "skip - port https://github.com/mapbox/mapbox-gl-js/pull/6263 - needs issue", - "render-tests/regressions/mapbox-gl-js#6706": "https://github.com/mapbox/mapbox-gl-native/issues/10619", - "render-tests/regressions/mapbox-gl-js#6806": "pending https://github.com/mapbox/mapbox-gl-js/pull/6812", - "render-tests/regressions/mapbox-gl-js#7271": "https://github.com/mapbox/mapbox-gl-native/issues/12888", - "render-tests/regressions/mapbox-gl-js#7302": "skip - js specific", - "render-tests/regressions/mapbox-gl-native#7357": "https://github.com/mapbox/mapbox-gl-native/issues/7357", - "render-tests/symbol-cross-fade/chinese": "https://github.com/mapbox/mapbox-gl-native/issues/10619", - "render-tests/video/default": "skip - https://github.com/mapbox/mapbox-gl-native/issues/601", - "render-tests/background-color/colorSpace-hcl": "needs issue", - "render-tests/text-variable-anchor/remember-last-placement": "skip - fails on gl-native, as symbol index is not functional at static map mode - needs issue", - "render-tests/regressions/mapbox-gl-js#8026": "skip - js specific", - "render-tests/fill-extrusion-geometry/linestring": "https://github.com/mapbox/mapbox-gl-native/pull/14240", - "render-tests/circle-sort-key/literal": "https://github.com/mapbox/mapbox-gl-native/issues/15008", - "render-tests/fill-sort-key/literal": "https://github.com/mapbox/mapbox-gl-native/issues/15008", - "render-tests/line-sort-key/literal": "https://github.com/mapbox/mapbox-gl-native/issues/15008", - "render-tests/regressions/mapbox-gl-js#8817": "skip - https://github.com/mapbox/mapbox-gl-native/issues/15737", - "render-tests/text-max-width/zero-width-point-placement": "https://github.com/mapbox/mapbox-gl-native/issues/15648", - "render-tests/icon-text-fit/text-variable-anchor-overlap": "https://github.com/mapbox/mapbox-gl-native/issues/15809", - "query-tests/fill-extrusion/base-in": "https://github.com/mapbox/mapbox-gl-native/issues/13139", - "query-tests/fill-extrusion/box-in": "https://github.com/mapbox/mapbox-gl-native/issues/13139", - "query-tests/fill-extrusion/side-in": "https://github.com/mapbox/mapbox-gl-native/issues/13139", - "query-tests/fill-extrusion/sort-concave-inner": "https://github.com/mapbox/mapbox-gl-native/issues/13139", - "query-tests/fill-extrusion/sort-concave-outer": "https://github.com/mapbox/mapbox-gl-native/issues/13139", - "query-tests/fill-extrusion/sort-rotated": "https://github.com/mapbox/mapbox-gl-native/issues/13139", - "query-tests/fill-extrusion/sort": "https://github.com/mapbox/mapbox-gl-native/issues/13139", - "query-tests/fill-extrusion/top-in": "https://github.com/mapbox/mapbox-gl-native/issues/13139", - "query-tests/regressions/mapbox-gl-js#7883": "https://github.com/mapbox/mapbox-gl-native/issues/14585", - "render-tests/fill-pattern/update-feature-state": "https://github.com/mapbox/mapbox-gl-native/issues/15895", - "render-tests/feature-state/promote-id": "https://github.com/mapbox/mapbox-gl-native/issues/16016", - "render-tests/icon-text-fit/stretch-fifteen-part": "https://github.com/mapbox/mapbox-gl-native/issues/16017", - "render-tests/icon-text-fit/stretch-nine-part-@2x": "https://github.com/mapbox/mapbox-gl-native/issues/16017", - "render-tests/icon-text-fit/stretch-nine-part-content-collision": "https://github.com/mapbox/mapbox-gl-native/issues/16017", - "render-tests/icon-text-fit/stretch-nine-part-content": "https://github.com/mapbox/mapbox-gl-native/issues/16017", - "render-tests/icon-text-fit/stretch-nine-part-just-height": "https://github.com/mapbox/mapbox-gl-native/issues/16017", - "render-tests/icon-text-fit/stretch-nine-part-just-width": "https://github.com/mapbox/mapbox-gl-native/issues/16017", - "render-tests/icon-text-fit/stretch-nine-part": "https://github.com/mapbox/mapbox-gl-native/issues/16017", - "render-tests/icon-text-fit/stretch-three-part": "https://github.com/mapbox/mapbox-gl-native/issues/16017", - "render-tests/icon-text-fit/stretch-two-part": "https://github.com/mapbox/mapbox-gl-native/issues/16017", - "render-tests/icon-text-fit/stretch-underscale": "https://github.com/mapbox/mapbox-gl-native/issues/16017", - "render-tests/regressions/mapbox-gl-js#9009": "https://github.com/mapbox/mapbox-gl-native/issues/16018", - "render-tests/zoomed-fill/negative-zoom": "https://github.com/mapbox/mapbox-gl-native/issues/16019", - "render-tests/runtime-styling/image-add-remove-add": "skip - https://github.com/mapbox/mapbox-gl-native/issues/16021", - "render-tests/runtime-styling/pattern-add-remove-add": "skip - https://github.com/mapbox/mapbox-gl-native/issues/16021", - "render-tests/text-size/nan": "https://github.com/mapbox/mapbox-gl-native/issues/16020" -} diff --git a/render-test/android/app/src/main/assets/to_zip.txt b/render-test/android/app/src/main/assets/to_zip.txt index c4ca0e97e..13f356723 100644 --- a/render-test/android/app/src/main/assets/to_zip.txt +++ b/render-test/android/app/src/main/assets/to_zip.txt @@ -12,8 +12,9 @@ mapbox-gl-js/test/integration/sprites/ vendor/mapbox-gl-styles/styles/ vendor/mapbox-gl-styles/sprites/ vendor/mvt-fixtures/real-world/ -render-test/ignores/ +metrics/expectations/ +metrics/ignores/ +metrics/tests/ metrics/next-android-render-test-runner/ -render-test/expected/ -android-manifest-probe-network-gfx.json -android-manifest-probe-memory.json +metrics/next-android-render-test-runner-metrics.json +metrics/next-android-render-test-runner-style.json diff --git a/render-test/ignores/android-temp-fail.json b/render-test/ignores/android-temp-fail.json deleted file mode 100644 index 655c69752..000000000 --- a/render-test/ignores/android-temp-fail.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "render-tests/icon-text-fit/both-text-anchor-1x-image-2x-screen": "Temporarily fails as android test app does not derefence symbolinks in sprites folder.", - "render-tests/icon-text-fit/both-text-anchor-2x-image-1x-screen": "Temporarily fails as android test app does not derefence symbolinks in sprites folder.", - "render-tests/regressions/mapbox-gl-native#9976": "Needs to be investigated and fixed.", - "render-tests/regressions/mapbox-gl-js#5911a": "Needs to be investigated and fixed.", - "render-tests/line-pattern/overscaled": "Needs to be investigated and fixed.", - "render-tests/runtime-styling/image-add-pattern": "Needs to be investigated and fixed.", - "render-tests/runtime-styling/image-update-pattern": "Needs to be investigated and fixed." -} diff --git a/render-test/ignores/linux-ignores.json b/render-test/ignores/linux-ignores.json deleted file mode 100644 index d5a1a85c6..000000000 --- a/render-test/ignores/linux-ignores.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "render-tests/regressions/mapbox-gl-js#7066": "Failing with mbgl-render-test", - "render-tests/regressions/mapbox-gl-js#5642": "Failing with mbgl-render-test", - "render-tests/line-pattern/opacity": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/15320", - "render-tests/fill-opacity/zoom-and-property-function-pattern": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/15322", - "render-tests/fill-pattern/literal": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/14423", - "render-tests/fill-pattern/opacity": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/14870", - "render-tests/fill-pattern/zoomed": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/14768", - "render-tests/line-translate/literal": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/14859" -} - diff --git a/render-test/linux-manifest.json b/render-test/linux-manifest.json deleted file mode 100644 index 91b062c2c..000000000 --- a/render-test/linux-manifest.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "base_test_path":"../mapbox-gl-js/test/integration", - "expectation_paths":["expected/"], - "ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"], - "vendor_path":"../vendor", - "asset_path": "../mapbox-gl-js/test/integration" -} \ No newline at end of file diff --git a/render-test/linux-probe-manifest.json b/render-test/linux-probe-manifest.json deleted file mode 100644 index f0ebeedd1..000000000 --- a/render-test/linux-probe-manifest.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "probe_test_path":".", - "expectation_paths":["expected/render-tests"], - "ignore_paths":["../render-test/tests/should-fail.json"], - "vendor_path":"../vendor", - "asset_path": "../mapbox-gl-js/test/integration" -} \ No newline at end of file diff --git a/render-test/linux-query-manifest.json b/render-test/linux-query-manifest.json deleted file mode 100644 index 6d2071b19..000000000 --- a/render-test/linux-query-manifest.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "base_test_path":"../mapbox-gl-js/test/integration", - "ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"], - "vendor_path":"../vendor", - "asset_path": "../mapbox-gl-js/test/integration" -} \ No newline at end of file diff --git a/render-test/mac-ignores.json b/render-test/mac-ignores.json deleted file mode 100644 index 87bef31c5..000000000 --- a/render-test/mac-ignores.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "render-tests/regressions/mapbox-gl-js#7066": "Failing with mbgl-render-test", - "render-tests/regressions/mapbox-gl-js#5642": "Failing with mbgl-render-test" -} - diff --git a/render-test/mac-manifest.json b/render-test/mac-manifest.json deleted file mode 100644 index 13a225cf4..000000000 --- a/render-test/mac-manifest.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "base_test_path":"../mapbox-gl-js/test/integration", - "expectation_paths":["expected/"], - "ignore_paths":["../platform/node/test/ignores.json", "../render-test/mac-ignores.json", "../render-test/tests/should-fail.json"], - "vendor_path":"../vendor", - "asset_path": "../mapbox-gl-js/test/integration" -} \ No newline at end of file diff --git a/render-test/mac-probe-manifest.json b/render-test/mac-probe-manifest.json deleted file mode 100644 index 154f6676d..000000000 --- a/render-test/mac-probe-manifest.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "probe_test_path":".", - "expectation_paths":["expected/"], - "ignore_paths":["../render-test/mac-ignores.json", "../render-test/tests/should-fail.json"], - "vendor_path":"../vendor", - "asset_path": "../mapbox-gl-js/test/integration", - "metric_path":"metrics/mac/" -} \ No newline at end of file diff --git a/render-test/metrics/mac/tests/memory/pass-memory-size-is-same/metrics.json b/render-test/metrics/mac/tests/memory/pass-memory-size-is-same/metrics.json deleted file mode 100644 index 0c6c8dc35..000000000 --- a/render-test/metrics/mac/tests/memory/pass-memory-size-is-same/metrics.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "memory": [ - [ - "after setZoom 0.9", - 21439, - 46 - ], - [ - "end", - 23385, - 91 - ], - [ - "start", - 0, - 0 - ] - ] -} diff --git a/render-test/tests/memory/pass-memory-size-is-same/expected.png b/render-test/tests/memory/pass-memory-size-is-same/expected.png deleted file mode 100644 index 0858c19f0521b74f4a058fbddc280ea48a5751bb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 96 zcmeAS@N?(olHy`uVBq!ia0vp^4j|0I1|(Ny7TyC=>Ygr+Ar*7p9%SSN@(wNd?w`wN fz5vKX1`XxR4h{^A{hv+-f`mO?{an^LB{Ts5qcarg diff --git a/render-test/tests/should-fail.json b/render-test/tests/should-fail.json deleted file mode 100644 index b539d6601..000000000 --- a/render-test/tests/should-fail.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "tests/file-size/fail-size-is-over": "Should fail, size is bigger than expected.", - "tests/file-size/fail-size-is-under": "Should fail, size is smaller than expected.", - "tests/file-size/fail-file-doesnt-match": "Should fail, doesn't match the expectation.", - "tests/file-size/fail-file-not-found": "Should fail, file not found.", - "tests/network/fail-requests": "Should fail, number of requests higher than expected.", - "tests/network/fail-transferred": "Should fail, amount of transferred data higher than expected.", - "tests/network/fail-requests-transferred": "Should fail, number of requests higher than expected and amount of transferred data less than expected.", - "tests/memory/fail-memory-size-is-too-big": "Should fail, memory size is bigger than expected.", - "tests/memory/fail-memory-size-is-too-small": "Should fail, memory size is smaller than expected.", - "tests/gfx/fail-ib-mem-mismatch": "Should fail, combined byte size of index buffers doesn't match the expectation.", - "tests/gfx/fail-negative-framebuffer-count": "Should fail, number of frame buffers is higher than expected.", - "tests/gfx/fail-texture-mem-mismatch": "Should fail, combined byte size of textures doesn't match the expectation.", - "tests/gfx/fail-too-many-drawcalls": "Should fail, number of draw calls higher than expected.", - "tests/gfx/fail-too-few-buffers": "Should fail, number of vertex and index buffers is smaller than expected.", - "tests/gfx/fail-too-few-textures": "Should fail, number of textures is smaller than expected.", - "tests/gfx/fail-vb-mem-mismatch": "Should fail, combined byte size of index buffers doesn't match the expectation." -}