diff --git a/.gitignore b/.gitignore index bf6b13e00..538870edf 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ *.gcno *.gcda *~ +**/actual.png +**/diff.png +render-test/index.html offline.db new_offline.db xcuserdata @@ -19,8 +22,6 @@ xcuserdata /test/fixtures/api/2.png /test/fixtures/offline_database/offline.db /test/fixtures/offline_database/offline.db-* -/test/fixtures/**/actual.png -/test/fixtures/**/diff.png /test/output /node_modules /platform/ios/benchmark/assets/glyphs/DIN* diff --git a/next/render-test/CMakeLists.txt b/next/render-test/CMakeLists.txt index 5aadd0f85..80709d2b8 100644 --- a/next/render-test/CMakeLists.txt +++ b/next/render-test/CMakeLists.txt @@ -47,8 +47,19 @@ add_test( render-tests --recycle-map --shuffle - --seed - ${MBGL_RENDER_TEST_SEED} + --seed=${MBGL_RENDER_TEST_SEED} + WORKING_DIRECTORY ${MBGL_ROOT} +) + +add_test( + NAME mbgl-render-test-probes + COMMAND + mbgl-render-test + render-tests + --recycle-map + --shuffle + --seed=${MBGL_RENDER_TEST_SEED} + --rootPath=render-test WORKING_DIRECTORY ${MBGL_ROOT} ) diff --git a/render-test/parser.cpp b/render-test/parser.cpp index 613fc566e..3da7e6ea0 100644 --- a/render-test/parser.cpp +++ b/render-test/parser.cpp @@ -421,15 +421,17 @@ std::vector> parseIgnores() { auto mainIgnoresPath = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("platform/node/test/ignores.json"); mbgl::filesystem::path platformSpecificIgnores; + mbgl::filesystem::path ownTestsIgnores = + mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("render-test/tests/should-fail.json"); #ifdef __APPLE__ platformSpecificIgnores = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("render-test/mac-ignores.json"); #elif __linux__ platformSpecificIgnores = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("render-test/linux-ignores.json"); #endif - - std::vector ignoresPaths = { mainIgnoresPath, platformSpecificIgnores }; - for (auto path: ignoresPaths) { + + std::vector ignoresPaths = {mainIgnoresPath, platformSpecificIgnores, ownTestsIgnores}; + for (const auto& path : ignoresPaths) { auto maybeIgnores = readJson(path); if (!maybeIgnores.is()) { continue; diff --git a/render-test/tests/filesize/fail-file-doesnt-match/expected.png b/render-test/tests/filesize/fail-file-doesnt-match/expected.png new file mode 100644 index 000000000..feed2e457 Binary files /dev/null and b/render-test/tests/filesize/fail-file-doesnt-match/expected.png differ diff --git a/render-test/tests/filesize/fail-file-doesnt-match/style.json b/render-test/tests/filesize/fail-file-doesnt-match/style.json new file mode 100644 index 000000000..91e90ffbe --- /dev/null +++ b/render-test/tests/filesize/fail-file-doesnt-match/style.json @@ -0,0 +1,36 @@ +{ + "version": 8, + "metadata": { + "test": { + "operations": [ + ["probeFileSize", "style", "aaaaa.json"], + ["probeFileSize", "image", "bbbbb.png"] + ], + "width": 64, + "height": 64 + } + }, + "sources": { + "geojson": { + "type": "geojson", + "data": { + "type": "Point", + "coordinates": [ + 0, + 0 + ] + } + } + }, + "sprite": "local://sprites/sprite", + "layers": [ + { + "id": "symbol", + "type": "symbol", + "source": "geojson", + "layout": { + "icon-image": "circle-12" + } + } + ] +} diff --git a/render-test/tests/filesize/fail-size-is-over/expected.png b/render-test/tests/filesize/fail-size-is-over/expected.png new file mode 100644 index 000000000..feed2e457 Binary files /dev/null and b/render-test/tests/filesize/fail-size-is-over/expected.png differ diff --git a/render-test/tests/filesize/fail-size-is-over/metrics.json b/render-test/tests/filesize/fail-size-is-over/metrics.json new file mode 100644 index 000000000..2df7e7da1 --- /dev/null +++ b/render-test/tests/filesize/fail-size-is-over/metrics.json @@ -0,0 +1 @@ +{"fileSize":[["image","../render-test/tests/filesize/fail-size-is-over/expected.png",100],["style","../render-test/tests/filesize/fail-size-is-over/style.json",100]]} diff --git a/render-test/tests/filesize/fail-size-is-over/style.json b/render-test/tests/filesize/fail-size-is-over/style.json new file mode 100644 index 000000000..ce2069a37 --- /dev/null +++ b/render-test/tests/filesize/fail-size-is-over/style.json @@ -0,0 +1,36 @@ +{ + "version": 8, + "metadata": { + "test": { + "operations": [ + ["probeFileSize", "style", "style.json"], + ["probeFileSize", "image", "expected.png"] + ], + "width": 64, + "height": 64 + } + }, + "sources": { + "geojson": { + "type": "geojson", + "data": { + "type": "Point", + "coordinates": [ + 0, + 0 + ] + } + } + }, + "sprite": "local://sprites/sprite", + "layers": [ + { + "id": "symbol", + "type": "symbol", + "source": "geojson", + "layout": { + "icon-image": "circle-12" + } + } + ] +} diff --git a/render-test/tests/filesize/fail-size-is-under/expected.png b/render-test/tests/filesize/fail-size-is-under/expected.png new file mode 100644 index 000000000..feed2e457 Binary files /dev/null and b/render-test/tests/filesize/fail-size-is-under/expected.png differ diff --git a/render-test/tests/filesize/fail-size-is-under/metrics.json b/render-test/tests/filesize/fail-size-is-under/metrics.json new file mode 100644 index 000000000..92d3f1e3d --- /dev/null +++ b/render-test/tests/filesize/fail-size-is-under/metrics.json @@ -0,0 +1 @@ +{"fileSize":[["image","../render-test/tests/filesize/fail-size-is-under/expected.png",999],["style","../render-test/tests/filesize/fail-size-is-under/style.json",999]]} diff --git a/render-test/tests/filesize/fail-size-is-under/style.json b/render-test/tests/filesize/fail-size-is-under/style.json new file mode 100644 index 000000000..ce2069a37 --- /dev/null +++ b/render-test/tests/filesize/fail-size-is-under/style.json @@ -0,0 +1,36 @@ +{ + "version": 8, + "metadata": { + "test": { + "operations": [ + ["probeFileSize", "style", "style.json"], + ["probeFileSize", "image", "expected.png"] + ], + "width": 64, + "height": 64 + } + }, + "sources": { + "geojson": { + "type": "geojson", + "data": { + "type": "Point", + "coordinates": [ + 0, + 0 + ] + } + } + }, + "sprite": "local://sprites/sprite", + "layers": [ + { + "id": "symbol", + "type": "symbol", + "source": "geojson", + "layout": { + "icon-image": "circle-12" + } + } + ] +} diff --git a/render-test/tests/filesize/pass-size-is-same/expected.png b/render-test/tests/filesize/pass-size-is-same/expected.png new file mode 100644 index 000000000..feed2e457 Binary files /dev/null and b/render-test/tests/filesize/pass-size-is-same/expected.png differ diff --git a/render-test/tests/filesize/pass-size-is-same/metrics.json b/render-test/tests/filesize/pass-size-is-same/metrics.json new file mode 100644 index 000000000..4c01d3b2c --- /dev/null +++ b/render-test/tests/filesize/pass-size-is-same/metrics.json @@ -0,0 +1 @@ +{"fileSize":[["image","../render-test/tests/filesize/pass-size-is-same/expected.png",686],["style","../render-test/tests/filesize/pass-size-is-same/style.json",609]]} \ No newline at end of file diff --git a/render-test/tests/filesize/pass-size-is-same/style.json b/render-test/tests/filesize/pass-size-is-same/style.json new file mode 100644 index 000000000..ce2069a37 --- /dev/null +++ b/render-test/tests/filesize/pass-size-is-same/style.json @@ -0,0 +1,36 @@ +{ + "version": 8, + "metadata": { + "test": { + "operations": [ + ["probeFileSize", "style", "style.json"], + ["probeFileSize", "image", "expected.png"] + ], + "width": 64, + "height": 64 + } + }, + "sources": { + "geojson": { + "type": "geojson", + "data": { + "type": "Point", + "coordinates": [ + 0, + 0 + ] + } + } + }, + "sprite": "local://sprites/sprite", + "layers": [ + { + "id": "symbol", + "type": "symbol", + "source": "geojson", + "layout": { + "icon-image": "circle-12" + } + } + ] +} diff --git a/render-test/tests/should-fail.json b/render-test/tests/should-fail.json new file mode 100644 index 000000000..28859d415 --- /dev/null +++ b/render-test/tests/should-fail.json @@ -0,0 +1,5 @@ +{ + "tests/filesize/fail-size-is-over": "Should fail, size is bigger than expected.", + "tests/filesize/fail-size-is-under": "Should fail, size is smaller than expected.", + "tests/filesize/fail-file-doesnt-match": "Should fail, file not found." +}