From 629c0e1142a382ada101e93e7c58bd2b1bce37bc Mon Sep 17 00:00:00 2001 From: "Thiago Marcos P. Santos" Date: Wed, 9 Oct 2019 22:00:30 +0300 Subject: [PATCH] [render-test] Add tests for filesize probe And make sure we run it on the bots --- .gitignore | 5 ++- next/render-test/CMakeLists.txt | 15 +++++++- render-test/parser.cpp | 8 ++-- .../fail-file-doesnt-match/expected.png | Bin 0 -> 686 bytes .../fail-file-doesnt-match/style.json | 36 ++++++++++++++++++ .../filesize/fail-size-is-over/expected.png | Bin 0 -> 686 bytes .../filesize/fail-size-is-over/metrics.json | 1 + .../filesize/fail-size-is-over/style.json | 36 ++++++++++++++++++ .../filesize/fail-size-is-under/expected.png | Bin 0 -> 686 bytes .../filesize/fail-size-is-under/metrics.json | 1 + .../filesize/fail-size-is-under/style.json | 36 ++++++++++++++++++ .../filesize/pass-size-is-same/expected.png | Bin 0 -> 686 bytes .../filesize/pass-size-is-same/metrics.json | 1 + .../filesize/pass-size-is-same/style.json | 36 ++++++++++++++++++ render-test/tests/should-fail.json | 5 +++ 15 files changed, 173 insertions(+), 7 deletions(-) create mode 100644 render-test/tests/filesize/fail-file-doesnt-match/expected.png create mode 100644 render-test/tests/filesize/fail-file-doesnt-match/style.json create mode 100644 render-test/tests/filesize/fail-size-is-over/expected.png create mode 100644 render-test/tests/filesize/fail-size-is-over/metrics.json create mode 100644 render-test/tests/filesize/fail-size-is-over/style.json create mode 100644 render-test/tests/filesize/fail-size-is-under/expected.png create mode 100644 render-test/tests/filesize/fail-size-is-under/metrics.json create mode 100644 render-test/tests/filesize/fail-size-is-under/style.json create mode 100644 render-test/tests/filesize/pass-size-is-same/expected.png create mode 100644 render-test/tests/filesize/pass-size-is-same/metrics.json create mode 100644 render-test/tests/filesize/pass-size-is-same/style.json create mode 100644 render-test/tests/should-fail.json diff --git a/.gitignore b/.gitignore index bf6b13e00..538870edf 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ *.gcno *.gcda *~ +**/actual.png +**/diff.png +render-test/index.html offline.db new_offline.db xcuserdata @@ -19,8 +22,6 @@ xcuserdata /test/fixtures/api/2.png /test/fixtures/offline_database/offline.db /test/fixtures/offline_database/offline.db-* -/test/fixtures/**/actual.png -/test/fixtures/**/diff.png /test/output /node_modules /platform/ios/benchmark/assets/glyphs/DIN* diff --git a/next/render-test/CMakeLists.txt b/next/render-test/CMakeLists.txt index 5aadd0f85..80709d2b8 100644 --- a/next/render-test/CMakeLists.txt +++ b/next/render-test/CMakeLists.txt @@ -47,8 +47,19 @@ add_test( render-tests --recycle-map --shuffle - --seed - ${MBGL_RENDER_TEST_SEED} + --seed=${MBGL_RENDER_TEST_SEED} + WORKING_DIRECTORY ${MBGL_ROOT} +) + +add_test( + NAME mbgl-render-test-probes + COMMAND + mbgl-render-test + render-tests + --recycle-map + --shuffle + --seed=${MBGL_RENDER_TEST_SEED} + --rootPath=render-test WORKING_DIRECTORY ${MBGL_ROOT} ) diff --git a/render-test/parser.cpp b/render-test/parser.cpp index 613fc566e..3da7e6ea0 100644 --- a/render-test/parser.cpp +++ b/render-test/parser.cpp @@ -421,15 +421,17 @@ std::vector> parseIgnores() { auto mainIgnoresPath = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("platform/node/test/ignores.json"); mbgl::filesystem::path platformSpecificIgnores; + mbgl::filesystem::path ownTestsIgnores = + mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("render-test/tests/should-fail.json"); #ifdef __APPLE__ platformSpecificIgnores = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("render-test/mac-ignores.json"); #elif __linux__ platformSpecificIgnores = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("render-test/linux-ignores.json"); #endif - - std::vector ignoresPaths = { mainIgnoresPath, platformSpecificIgnores }; - for (auto path: ignoresPaths) { + + std::vector ignoresPaths = {mainIgnoresPath, platformSpecificIgnores, ownTestsIgnores}; + for (const auto& path : ignoresPaths) { auto maybeIgnores = readJson(path); if (!maybeIgnores.is()) { continue; diff --git a/render-test/tests/filesize/fail-file-doesnt-match/expected.png b/render-test/tests/filesize/fail-file-doesnt-match/expected.png new file mode 100644 index 0000000000000000000000000000000000000000..feed2e457f16b98a90e0e3e49ee150c0fef0a566 GIT binary patch literal 686 zcmeAS@N?(olHy`uVBq!ia0vp^4j|0I1|(Ny7TyC=Or9=|Ar*{or0OmKIq4wY+k=ML zb5muGJ-q*U-sBEPrwe?AMVmBd#$OD4F3ggyIBBKGk-S%8Tz-5`E(JVd_Kov8ChyST zN`I-;#`ow-ldDsw=$#FU_S>V8nmrUeVjOe+sL2{;V)F=s9pB z!&-6qfkPFBwL9uQU0yr={br?J(OusjZvVa3w=XI?+QxCxW*77Ny6;zSuW?9booD~& z(Wz&b&&%*1PU){daqZ8;`On$E=k5L1(8m^*P+a@CqQO>Y!vQ947?ISl_w3i+WUJX1 zZf@n@cTc|Un;fUg1P3N0Vr}8qQ`!0dK0TOy{{4lM4(_|#Kj%(Th8ch`s_99__kw-C z>ddmbv(t`5Z|rc@NfPXiIHDv5XEs%2-!xe~xv`I}`UpZJf|azv-D>8giIWRA?9fqH zp6m1SUTlBA->UxI)SG+7?kO~~2zj6o7rtKZ&#Q4)7q&b5$K3>l`VQ@(z`j%gmA1AlD{+|~s nC&$}Y|F%O!KZx~n{a;?+R}(F?``*<76Ci`9tDnm{r-UW|9%wk1 literal 0 HcmV?d00001 diff --git a/render-test/tests/filesize/fail-file-doesnt-match/style.json b/render-test/tests/filesize/fail-file-doesnt-match/style.json new file mode 100644 index 000000000..91e90ffbe --- /dev/null +++ b/render-test/tests/filesize/fail-file-doesnt-match/style.json @@ -0,0 +1,36 @@ +{ + "version": 8, + "metadata": { + "test": { + "operations": [ + ["probeFileSize", "style", "aaaaa.json"], + ["probeFileSize", "image", "bbbbb.png"] + ], + "width": 64, + "height": 64 + } + }, + "sources": { + "geojson": { + "type": "geojson", + "data": { + "type": "Point", + "coordinates": [ + 0, + 0 + ] + } + } + }, + "sprite": "local://sprites/sprite", + "layers": [ + { + "id": "symbol", + "type": "symbol", + "source": "geojson", + "layout": { + "icon-image": "circle-12" + } + } + ] +} diff --git a/render-test/tests/filesize/fail-size-is-over/expected.png b/render-test/tests/filesize/fail-size-is-over/expected.png new file mode 100644 index 0000000000000000000000000000000000000000..feed2e457f16b98a90e0e3e49ee150c0fef0a566 GIT binary patch literal 686 zcmeAS@N?(olHy`uVBq!ia0vp^4j|0I1|(Ny7TyC=Or9=|Ar*{or0OmKIq4wY+k=ML zb5muGJ-q*U-sBEPrwe?AMVmBd#$OD4F3ggyIBBKGk-S%8Tz-5`E(JVd_Kov8ChyST zN`I-;#`ow-ldDsw=$#FU_S>V8nmrUeVjOe+sL2{;V)F=s9pB z!&-6qfkPFBwL9uQU0yr={br?J(OusjZvVa3w=XI?+QxCxW*77Ny6;zSuW?9booD~& z(Wz&b&&%*1PU){daqZ8;`On$E=k5L1(8m^*P+a@CqQO>Y!vQ947?ISl_w3i+WUJX1 zZf@n@cTc|Un;fUg1P3N0Vr}8qQ`!0dK0TOy{{4lM4(_|#Kj%(Th8ch`s_99__kw-C z>ddmbv(t`5Z|rc@NfPXiIHDv5XEs%2-!xe~xv`I}`UpZJf|azv-D>8giIWRA?9fqH zp6m1SUTlBA->UxI)SG+7?kO~~2zj6o7rtKZ&#Q4)7q&b5$K3>l`VQ@(z`j%gmA1AlD{+|~s nC&$}Y|F%O!KZx~n{a;?+R}(F?``*<76Ci`9tDnm{r-UW|9%wk1 literal 0 HcmV?d00001 diff --git a/render-test/tests/filesize/fail-size-is-over/metrics.json b/render-test/tests/filesize/fail-size-is-over/metrics.json new file mode 100644 index 000000000..2df7e7da1 --- /dev/null +++ b/render-test/tests/filesize/fail-size-is-over/metrics.json @@ -0,0 +1 @@ +{"fileSize":[["image","../render-test/tests/filesize/fail-size-is-over/expected.png",100],["style","../render-test/tests/filesize/fail-size-is-over/style.json",100]]} diff --git a/render-test/tests/filesize/fail-size-is-over/style.json b/render-test/tests/filesize/fail-size-is-over/style.json new file mode 100644 index 000000000..ce2069a37 --- /dev/null +++ b/render-test/tests/filesize/fail-size-is-over/style.json @@ -0,0 +1,36 @@ +{ + "version": 8, + "metadata": { + "test": { + "operations": [ + ["probeFileSize", "style", "style.json"], + ["probeFileSize", "image", "expected.png"] + ], + "width": 64, + "height": 64 + } + }, + "sources": { + "geojson": { + "type": "geojson", + "data": { + "type": "Point", + "coordinates": [ + 0, + 0 + ] + } + } + }, + "sprite": "local://sprites/sprite", + "layers": [ + { + "id": "symbol", + "type": "symbol", + "source": "geojson", + "layout": { + "icon-image": "circle-12" + } + } + ] +} diff --git a/render-test/tests/filesize/fail-size-is-under/expected.png b/render-test/tests/filesize/fail-size-is-under/expected.png new file mode 100644 index 0000000000000000000000000000000000000000..feed2e457f16b98a90e0e3e49ee150c0fef0a566 GIT binary patch literal 686 zcmeAS@N?(olHy`uVBq!ia0vp^4j|0I1|(Ny7TyC=Or9=|Ar*{or0OmKIq4wY+k=ML zb5muGJ-q*U-sBEPrwe?AMVmBd#$OD4F3ggyIBBKGk-S%8Tz-5`E(JVd_Kov8ChyST zN`I-;#`ow-ldDsw=$#FU_S>V8nmrUeVjOe+sL2{;V)F=s9pB z!&-6qfkPFBwL9uQU0yr={br?J(OusjZvVa3w=XI?+QxCxW*77Ny6;zSuW?9booD~& z(Wz&b&&%*1PU){daqZ8;`On$E=k5L1(8m^*P+a@CqQO>Y!vQ947?ISl_w3i+WUJX1 zZf@n@cTc|Un;fUg1P3N0Vr}8qQ`!0dK0TOy{{4lM4(_|#Kj%(Th8ch`s_99__kw-C z>ddmbv(t`5Z|rc@NfPXiIHDv5XEs%2-!xe~xv`I}`UpZJf|azv-D>8giIWRA?9fqH zp6m1SUTlBA->UxI)SG+7?kO~~2zj6o7rtKZ&#Q4)7q&b5$K3>l`VQ@(z`j%gmA1AlD{+|~s nC&$}Y|F%O!KZx~n{a;?+R}(F?``*<76Ci`9tDnm{r-UW|9%wk1 literal 0 HcmV?d00001 diff --git a/render-test/tests/filesize/fail-size-is-under/metrics.json b/render-test/tests/filesize/fail-size-is-under/metrics.json new file mode 100644 index 000000000..92d3f1e3d --- /dev/null +++ b/render-test/tests/filesize/fail-size-is-under/metrics.json @@ -0,0 +1 @@ +{"fileSize":[["image","../render-test/tests/filesize/fail-size-is-under/expected.png",999],["style","../render-test/tests/filesize/fail-size-is-under/style.json",999]]} diff --git a/render-test/tests/filesize/fail-size-is-under/style.json b/render-test/tests/filesize/fail-size-is-under/style.json new file mode 100644 index 000000000..ce2069a37 --- /dev/null +++ b/render-test/tests/filesize/fail-size-is-under/style.json @@ -0,0 +1,36 @@ +{ + "version": 8, + "metadata": { + "test": { + "operations": [ + ["probeFileSize", "style", "style.json"], + ["probeFileSize", "image", "expected.png"] + ], + "width": 64, + "height": 64 + } + }, + "sources": { + "geojson": { + "type": "geojson", + "data": { + "type": "Point", + "coordinates": [ + 0, + 0 + ] + } + } + }, + "sprite": "local://sprites/sprite", + "layers": [ + { + "id": "symbol", + "type": "symbol", + "source": "geojson", + "layout": { + "icon-image": "circle-12" + } + } + ] +} diff --git a/render-test/tests/filesize/pass-size-is-same/expected.png b/render-test/tests/filesize/pass-size-is-same/expected.png new file mode 100644 index 0000000000000000000000000000000000000000..feed2e457f16b98a90e0e3e49ee150c0fef0a566 GIT binary patch literal 686 zcmeAS@N?(olHy`uVBq!ia0vp^4j|0I1|(Ny7TyC=Or9=|Ar*{or0OmKIq4wY+k=ML zb5muGJ-q*U-sBEPrwe?AMVmBd#$OD4F3ggyIBBKGk-S%8Tz-5`E(JVd_Kov8ChyST zN`I-;#`ow-ldDsw=$#FU_S>V8nmrUeVjOe+sL2{;V)F=s9pB z!&-6qfkPFBwL9uQU0yr={br?J(OusjZvVa3w=XI?+QxCxW*77Ny6;zSuW?9booD~& z(Wz&b&&%*1PU){daqZ8;`On$E=k5L1(8m^*P+a@CqQO>Y!vQ947?ISl_w3i+WUJX1 zZf@n@cTc|Un;fUg1P3N0Vr}8qQ`!0dK0TOy{{4lM4(_|#Kj%(Th8ch`s_99__kw-C z>ddmbv(t`5Z|rc@NfPXiIHDv5XEs%2-!xe~xv`I}`UpZJf|azv-D>8giIWRA?9fqH zp6m1SUTlBA->UxI)SG+7?kO~~2zj6o7rtKZ&#Q4)7q&b5$K3>l`VQ@(z`j%gmA1AlD{+|~s nC&$}Y|F%O!KZx~n{a;?+R}(F?``*<76Ci`9tDnm{r-UW|9%wk1 literal 0 HcmV?d00001 diff --git a/render-test/tests/filesize/pass-size-is-same/metrics.json b/render-test/tests/filesize/pass-size-is-same/metrics.json new file mode 100644 index 000000000..4c01d3b2c --- /dev/null +++ b/render-test/tests/filesize/pass-size-is-same/metrics.json @@ -0,0 +1 @@ +{"fileSize":[["image","../render-test/tests/filesize/pass-size-is-same/expected.png",686],["style","../render-test/tests/filesize/pass-size-is-same/style.json",609]]} \ No newline at end of file diff --git a/render-test/tests/filesize/pass-size-is-same/style.json b/render-test/tests/filesize/pass-size-is-same/style.json new file mode 100644 index 000000000..ce2069a37 --- /dev/null +++ b/render-test/tests/filesize/pass-size-is-same/style.json @@ -0,0 +1,36 @@ +{ + "version": 8, + "metadata": { + "test": { + "operations": [ + ["probeFileSize", "style", "style.json"], + ["probeFileSize", "image", "expected.png"] + ], + "width": 64, + "height": 64 + } + }, + "sources": { + "geojson": { + "type": "geojson", + "data": { + "type": "Point", + "coordinates": [ + 0, + 0 + ] + } + } + }, + "sprite": "local://sprites/sprite", + "layers": [ + { + "id": "symbol", + "type": "symbol", + "source": "geojson", + "layout": { + "icon-image": "circle-12" + } + } + ] +} diff --git a/render-test/tests/should-fail.json b/render-test/tests/should-fail.json new file mode 100644 index 000000000..28859d415 --- /dev/null +++ b/render-test/tests/should-fail.json @@ -0,0 +1,5 @@ +{ + "tests/filesize/fail-size-is-over": "Should fail, size is bigger than expected.", + "tests/filesize/fail-size-is-under": "Should fail, size is smaller than expected.", + "tests/filesize/fail-file-doesnt-match": "Should fail, file not found." +}