tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[test-runner] Make Android RenderTestRunner bot fail when test cases are not fully passed (#15994)

Browse Source 
* [test-runner] Fix failed tests, generate android specific expectations

* [test-runner] Check render test runner testing result

* [test-runner] Add error case to android ignore list

*  [test-runner] Update metrics

* [test-runner] Move Android render tests with big diff(>0.006) to ignore list
This commit is contained in:
zmiao
2019-12-04 18:10:58 +02:00
committed by GitHub
parent 1e2cd1e086
commit e619552a5d
31 changed files with 357 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
render-test/android-manifest-probe-memory.json
View File

@@ -1,7 +1,7 @@
{
"base_test_path":"mapbox-gl-js/test/integration",
"expectation_paths":["render-test/expected"],
"ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"],
"expectation_paths":["render-test/expected", "render-test/expected/android"],
"ignore_paths":["render-test/ignores/ignores.json", "render-test/ignores/linux-ignores.json", "render-test/ignores/android-temp-fail.json"],
"vendor_path":"vendor",
"asset_path": "mapbox-gl-js/test/integration",
"result_path":"/sdcard/",

4
render-test/android-manifest-probe-network-gfx.json
View File

@@ -1,7 +1,7 @@
{
"base_test_path":"mapbox-gl-js/test/integration",
"expectation_paths":["render-test/expected"],
"ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"],
"expectation_paths":["render-test/expected", "render-test/expected/android"],
"ignore_paths":["render-test/ignores/ignores.json", "render-test/ignores/linux-ignores.json", "render-test/ignores/android-temp-fail.json"],
"vendor_path":"vendor",
"asset_path": "mapbox-gl-js/test/integration",
"result_path":"/sdcard/",

6
render-test/android-manifest.json
View File

@@ -1,10 +1,10 @@
{
"base_test_path":"mapbox-gl-js/test/integration",
"expectation_paths":["render-test/expected"],
"expectation_paths":["render-test/expected", "render-test/expected/android"],
"ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"],
"vendor_path":"vendor",
"asset_path": "mapbox-gl-js/test/integration",
"result_path":"/sdcard/",
"result_path": "/sdcard/",
"probes": ["probeGFX", "probeNetwork"],
"metric_path":"render-test/metrics/android"
}
}

5
render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java
View File

@@ -6,6 +6,7 @@ import androidx.test.rule.ActivityTestRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.Assert;
import org.junit.runner.RunWith;
import android.util.Log;
@@ -24,6 +25,8 @@ public class NativeActivityTest {
Log.v("Test", "Test is running");
Thread.sleep(1000L);
}
Log.v("Test", "End the test");
Log.v("Test", "All render tests are finished!");
Assert.assertTrue("All test cases are passed", TestState.testResult);
}
}

1
render-test/android/app/src/androidTest/java/android/app/TestState.java
View File

@@ -2,4 +2,5 @@ package android.app;
public class TestState {
static boolean running = true;
static boolean testResult = true;
}

4
render-test/android/app/src/main/assets/to_zip.txt
View File

@@ -12,8 +12,8 @@ mapbox-gl-js/test/integration/sprites/
vendor/mapbox-gl-styles/styles/
vendor/mapbox-gl-styles/sprites/
vendor/mvt-fixtures/real-world/
ignores/
render-test/ignores/
render-test/metrics/android/
render-test/expected/
android-manifest-probe-network-gfx.json
android-manifest-probe-memory.json
android-manifest-probe-memory.json

BIN
render-test/expected/android/render-tests/combinations/symbol-translucent--fill-translucent/expected.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 429 B

BIN
render-test/expected/android/render-tests/regressions/mapbox-gl-js#2305/expected.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 51 KiB

BIN
render-test/expected/android/render-tests/regressions/mapbox-gl-js#5978/expected.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 731 B

BIN
render-test/expected/android/render-tests/sprites/1x-screen-1x-pattern/expected.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.6 KiB

BIN
render-test/expected/android/render-tests/sprites/1x-screen-2x-pattern/expected.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.2 KiB

BIN
render-test/expected/android/render-tests/sprites/2x-screen-1x-pattern/expected.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 6.2 KiB

BIN
render-test/expected/android/render-tests/sprites/2x-screen-2x-pattern/expected.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.2 KiB

9
render-test/ignores/android-temp-fail.json Normal file
View File

@@ -0,0 +1,9 @@
{
"render-tests/icon-text-fit/both-text-anchor-1x-image-2x-screen": "Temporarily fails as android test app does not derefence symbolinks in sprites folder.",
"render-tests/icon-text-fit/both-text-anchor-2x-image-1x-screen": "Temporarily fails as android test app does not derefence symbolinks in sprites folder.",
"render-tests/regressions/mapbox-gl-native#9976": "Needs to be investigated and fixed.",
"render-tests/regressions/mapbox-gl-js#5911a": "Needs to be investigated and fixed.",
"render-tests/line-pattern/overscaled": "Needs to be investigated and fixed.",
"render-tests/runtime-styling/image-add-pattern": "Needs to be investigated and fixed.",
"render-tests/runtime-styling/image-update-pattern": "Needs to be investigated and fixed."
}

0
render-test/linux-ignores.json → render-test/ignores/linux-ignores.json
View File

2
render-test/linux-manifest-probe-gfx-network.json
View File

@@ -1,7 +1,7 @@
{
"base_test_path":"../mapbox-gl-js/test/integration",
"expectation_paths":["expected/"],
"ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"],
"ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"],
"vendor_path":"../vendor",
"asset_path": "../mapbox-gl-js/test/integration",
"probes": ["probeGFX", "probeNetwork"],

2
render-test/linux-manifest-probe-memory.json
View File

@@ -1,7 +1,7 @@
{
"base_test_path":"../mapbox-gl-js/test/integration",
"expectation_paths":["expected/"],
"ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"],
"ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"],
"vendor_path":"../vendor",
"asset_path": "../mapbox-gl-js/test/integration",
"probes": ["probeMemory"],

2
render-test/linux-manifest.json
View File

@@ -1,7 +1,7 @@
{
"base_test_path":"../mapbox-gl-js/test/integration",
"expectation_paths":["expected/"],
"ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"],
"ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"],
"vendor_path":"../vendor",
"asset_path": "../mapbox-gl-js/test/integration"
}

2
render-test/linux-query-manifest.json
View File

@@ -1,6 +1,6 @@
{
"base_test_path":"../mapbox-gl-js/test/integration",
"ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"],
"ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"],
"vendor_path":"../vendor",
"asset_path": "../mapbox-gl-js/test/integration"
}

35
render-test/metrics/android/render-tests/line-dasharray/round/segments/metrics.json Normal file
View File

@@ -0,0 +1,35 @@
{
"network": [
[
"probeNetwork - default - end",
0,
0
],
[
"probeNetwork - default - start",
0,
0
]
],
"gfx": [
[
"probeGFX - default - end",
2,
4,
9,
1,
[
36864,
36864
],
[
202,
202
],
[
704,
704
]
]
]
}

35
render-test/metrics/android/render-tests/line-dasharray/round/zero-gap-width/metrics.json Normal file
View File

@@ -0,0 +1,35 @@
{
"network": [
[
"probeNetwork - default - end",
0,
0
],
[
"probeNetwork - default - start",
0,
0
]
],
"gfx": [
[
"probeGFX - default - end",
2,
4,
9,
1,
[
36864,
36864
],
[
202,
202
],
[
704,
704
]
]
]
}

35
render-test/metrics/android/render-tests/text-field/formatted-images-constant-size/metrics.json Normal file
View File

@@ -0,0 +1,35 @@
{
"network": [
[
"probeNetwork - default - end",
3,
161976
],
[
"probeNetwork - default - start",
0,
0
]
],
"gfx": [
[
"probeGFX - default - end",
2,
4,
13,
1,
[
24580,
24580
],
[
190,
190
],
[
1856,
1856
]
]
]
}

35
render-test/metrics/android/render-tests/text-field/formatted-images-line/metrics.json Normal file
View File

@@ -0,0 +1,35 @@
{
"network": [
[
"probeNetwork - default - end",
3,
120865
],
[
"probeNetwork - default - start",
0,
0
]
],
"gfx": [
[
"probeGFX - default - end",
3,
4,
11,
1,
[
28318,
28318
],
[
190,
190
],
[
1424,
1424
]
]
]
}

35
render-test/metrics/android/render-tests/text-field/formatted-images-multiline/metrics.json Normal file
View File

@@ -0,0 +1,35 @@
{
"network": [
[
"probeNetwork - default - end",
3,
120865
],
[
"probeNetwork - default - start",
0,
0
]
],
"gfx": [
[
"probeGFX - default - end",
2,
6,
13,
1,
[
44921,
44921
],
[
310,
310
],
[
3136,
3136
]
]
]
}

35
render-test/metrics/android/render-tests/text-field/formatted-images-variable-anchors-justification/metrics.json Normal file
View File

@@ -0,0 +1,35 @@
{
"network": [
[
"probeNetwork - default - end",
3,
161976
],
[
"probeNetwork - default - start",
0,
0
]
],
"gfx": [
[
"probeGFX - default - end",
1,
4,
9,
1,
[
24580,
24580
],
[
2290,
2290
],
[
24256,
24256
]
]
]
}

35
render-test/metrics/android/render-tests/text-field/formatted-images-vertical/metrics.json Normal file
View File

@@ -0,0 +1,35 @@
{
"network": [
[
"probeNetwork - default - end",
4,
255564
],
[
"probeNetwork - default - start",
0,
0
]
],
"gfx": [
[
"probeGFX - default - end",
1,
4,
9,
1,
[
35756,
35756
],
[
310,
310
],
[
3136,
3136
]
]
]
}

35
render-test/metrics/android/render-tests/text-field/formatted-images-zoom-dependent-size/metrics.json Normal file
View File

@@ -0,0 +1,35 @@
{
"network": [
[
"probeNetwork - default - end",
3,
161976
],
[
"probeNetwork - default - start",
0,
0
]
],
"gfx": [
[
"probeGFX - default - end",
2,
13,
13,
1,
[
98636,
98636
],
[
142,
142
],
[
1344,
1344
]
]
]
}

35
render-test/metrics/android/render-tests/text-field/formatted-images/metrics.json Normal file
View File

@@ -0,0 +1,35 @@
{
"network": [
[
"probeNetwork - default - end",
3,
161976
],
[
"probeNetwork - default - start",
0,
0
]
],
"gfx": [
[
"probeGFX - default - end",
3,
4,
17,
1,
[
52510,
52510
],
[
166,
166
],
[
1600,
1600
]
]
]
}