tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[test-runner] Make Android RenderTestRunner bot fail when test cases are not fully passed (#15994)

Browse Source 
* [test-runner] Fix failed tests, generate android specific expectations

* [test-runner] Check render test runner testing result

* [test-runner] Add error case to android ignore list

*  [test-runner] Update metrics

* [test-runner] Move Android render tests with big diff(>0.006) to ignore list
This commit is contained in:
zmiao
2019-12-04 18:10:58 +02:00
committed by GitHub
parent 1e2cd1e086
commit e619552a5d
31 changed files with 357 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
circle.yml
View File

@@ -665,6 +665,12 @@ jobs:
--device-ids flame --os-version-ids 29 --locales en --orientations portrait --timeout 20m \
--directories-to-pull /sdcard --results-dir render-test-app-${CIRCLE_BUILD_NUM} \
--no-record-video --no-performance-metrics
fi
- run:
name: Retrieve TestResult from gcloud
when: always
command: |
if [[ -n "${GCLOUD_SERVICE_ACCOUNT_JSON}" && -z "${SKIP_FIREBASE:-}" ]]; then
testResult=$(gsutil ls gs://test-lab-186672a0qp5bq-ycr70axads3nc/render-test-app-${CIRCLE_BUILD_NUM}/**/*.html)
mkdir -p /tmp/tests/render-test-app
gsutil cp $testResult /tmp/tests/render-test-app | true

15
next/platform/android/android.cmake
View File

@@ -357,26 +357,15 @@ add_custom_command(
-E
copy
${MBGL_ROOT}/platform/node/test/ignores.json
${MBGL_ROOT}/ignores/ignores.json
COMMAND
${CMAKE_COMMAND}
-E
copy
${MBGL_ROOT}/render-test/linux-ignores.json
${MBGL_ROOT}/ignores/linux-ignores.json
${MBGL_ROOT}/render-test/ignores/ignores.json
COMMAND
${CMAKE_COMMAND}
-E
tar
"chvf"
"chf"
"render-test/android/app/src/main/assets/data.zip"
--format=zip
--files-from=render-test/android/app/src/main/assets/to_zip.txt
COMMAND
${CMAKE_COMMAND}
-E
remove_directory
${MBGL_ROOT}/ignores
COMMAND
${CMAKE_COMMAND}
-E

11
platform/android/src/test/render_test_runner.cpp
View File

@@ -79,7 +79,7 @@ std::string jstringToStdString(JNIEnv* env, jstring jStr) {
return ret;
}
void changeState(JNIEnv* env, struct android_app* app) {
void changeState(JNIEnv* env, struct android_app* app, bool result) {
jobject nativeActivity = app->activity->clazz;
jclass acl = env->GetObjectClass(nativeActivity);
jmethodID getClassLoader = env->GetMethodID(acl, "getClassLoader", "()Ljava/lang/ClassLoader;");
@@ -89,6 +89,8 @@ void changeState(JNIEnv* env, struct android_app* app) {
JavaWrapper<jstring> strClassName(env, env->NewStringUTF("android.app.TestState"));
jclass testStateClass = static_cast<jclass>(env->CallObjectMethod(cls, findClass, strClassName.get()));
if (testStateClass != NULL) {
jfieldID id0 = env->GetStaticFieldID(testStateClass, "testResult", "Z");
env->SetStaticBooleanField(testStateClass, id0, result);
jfieldID id = env->GetStaticFieldID(testStateClass, "running", "Z");
env->SetStaticBooleanField(testStateClass, id, false);
}
@@ -278,10 +280,11 @@ void android_main(struct android_app* app) {
mbgl::Log::Info(mbgl::Event::General, "End running RenderTestRunner with manifest: '%s'", manifest.c_str());
return result;
};
runTestWithManifest("/android-manifest-probe-network-gfx.json");
runTestWithManifest("/android-manifest-probe-memory.json");
auto result = runTestWithManifest("/android-manifest-probe-network-gfx.json");
result = runTestWithManifest("/android-manifest-probe-memory.json") && result;
mbgl::Log::Info(mbgl::Event::General, "All tests are finished!");
changeState(env, app);
changeState(env, app, result);
}
while (true) {
ALooper_pollAll(0, &outFd, &outEvents, reinterpret_cast<void**>(&source));

4
render-test/android-manifest-probe-memory.json
View File

@@ -1,7 +1,7 @@
{
"base_test_path":"mapbox-gl-js/test/integration",
"expectation_paths":["render-test/expected"],
"ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"],
"expectation_paths":["render-test/expected", "render-test/expected/android"],
"ignore_paths":["render-test/ignores/ignores.json", "render-test/ignores/linux-ignores.json", "render-test/ignores/android-temp-fail.json"],
"vendor_path":"vendor",
"asset_path": "mapbox-gl-js/test/integration",
"result_path":"/sdcard/",

4
render-test/android-manifest-probe-network-gfx.json
View File

@@ -1,7 +1,7 @@
{
"base_test_path":"mapbox-gl-js/test/integration",
"expectation_paths":["render-test/expected"],
"ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"],
"expectation_paths":["render-test/expected", "render-test/expected/android"],
"ignore_paths":["render-test/ignores/ignores.json", "render-test/ignores/linux-ignores.json", "render-test/ignores/android-temp-fail.json"],
"vendor_path":"vendor",
"asset_path": "mapbox-gl-js/test/integration",
"result_path":"/sdcard/",

6
render-test/android-manifest.json
View File

@@ -1,10 +1,10 @@
{
"base_test_path":"mapbox-gl-js/test/integration",
"expectation_paths":["render-test/expected"],
"expectation_paths":["render-test/expected", "render-test/expected/android"],
"ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"],
"vendor_path":"vendor",
"asset_path": "mapbox-gl-js/test/integration",
"result_path":"/sdcard/",
"result_path": "/sdcard/",
"probes": ["probeGFX", "probeNetwork"],
"metric_path":"render-test/metrics/android"
}
}

5
render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java
View File

@@ -6,6 +6,7 @@ import androidx.test.rule.ActivityTestRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.Assert;
import org.junit.runner.RunWith;
import android.util.Log;
@@ -24,6 +25,8 @@ public class NativeActivityTest {
Log.v("Test", "Test is running");
Thread.sleep(1000L);
}
Log.v("Test", "End the test");
Log.v("Test", "All render tests are finished!");
Assert.assertTrue("All test cases are passed", TestState.testResult);
}
}

1
render-test/android/app/src/androidTest/java/android/app/TestState.java
View File

@@ -2,4 +2,5 @@ package android.app;
public class TestState {
static boolean running = true;
static boolean testResult = true;
}

4
render-test/android/app/src/main/assets/to_zip.txt
View File

@@ -12,8 +12,8 @@ mapbox-gl-js/test/integration/sprites/
vendor/mapbox-gl-styles/styles/
vendor/mapbox-gl-styles/sprites/
vendor/mvt-fixtures/real-world/
ignores/
render-test/ignores/
render-test/metrics/android/
render-test/expected/
android-manifest-probe-network-gfx.json
android-manifest-probe-memory.json
android-manifest-probe-memory.json

BIN
render-test/expected/android/render-tests/combinations/symbol-translucent--fill-translucent/expected.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 429 B

BIN
render-test/expected/android/render-tests/regressions/mapbox-gl-js#2305/expected.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 51 KiB

BIN
render-test/expected/android/render-tests/regressions/mapbox-gl-js#5978/expected.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 731 B

BIN
render-test/expected/android/render-tests/sprites/1x-screen-1x-pattern/expected.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.6 KiB

BIN
render-test/expected/android/render-tests/sprites/1x-screen-2x-pattern/expected.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.2 KiB

BIN
render-test/expected/android/render-tests/sprites/2x-screen-1x-pattern/expected.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 6.2 KiB

BIN
render-test/expected/android/render-tests/sprites/2x-screen-2x-pattern/expected.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.2 KiB

9
render-test/ignores/android-temp-fail.json Normal file
View File

@@ -0,0 +1,9 @@
{
"render-tests/icon-text-fit/both-text-anchor-1x-image-2x-screen": "Temporarily fails as android test app does not derefence symbolinks in sprites folder.",
"render-tests/icon-text-fit/both-text-anchor-2x-image-1x-screen": "Temporarily fails as android test app does not derefence symbolinks in sprites folder.",
"render-tests/regressions/mapbox-gl-native#9976": "Needs to be investigated and fixed.",
"render-tests/regressions/mapbox-gl-js#5911a": "Needs to be investigated and fixed.",
"render-tests/line-pattern/overscaled": "Needs to be investigated and fixed.",
"render-tests/runtime-styling/image-add-pattern": "Needs to be investigated and fixed.",
"render-tests/runtime-styling/image-update-pattern": "Needs to be investigated and fixed."
}

0
render-test/linux-ignores.json → render-test/ignores/linux-ignores.json
View File

2
render-test/linux-manifest-probe-gfx-network.json
View File

@@ -1,7 +1,7 @@
{
"base_test_path":"../mapbox-gl-js/test/integration",
"expectation_paths":["expected/"],
"ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"],
"ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"],
"vendor_path":"../vendor",
"asset_path": "../mapbox-gl-js/test/integration",
"probes": ["probeGFX", "probeNetwork"],

2
render-test/linux-manifest-probe-memory.json
View File

@@ -1,7 +1,7 @@
{
"base_test_path":"../mapbox-gl-js/test/integration",
"expectation_paths":["expected/"],
"ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"],
"ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"],
"vendor_path":"../vendor",
"asset_path": "../mapbox-gl-js/test/integration",
"probes": ["probeMemory"],

2
render-test/linux-manifest.json
View File

@@ -1,7 +1,7 @@
{
"base_test_path":"../mapbox-gl-js/test/integration",
"expectation_paths":["expected/"],
"ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"],
"ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"],
"vendor_path":"../vendor",
"asset_path": "../mapbox-gl-js/test/integration"
}

2
render-test/linux-query-manifest.json
View File

@@ -1,6 +1,6 @@
{
"base_test_path":"../mapbox-gl-js/test/integration",
"ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"],
"ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"],
"vendor_path":"../vendor",
"asset_path": "../mapbox-gl-js/test/integration"
}

35
render-test/metrics/android/render-tests/line-dasharray/round/segments/metrics.json Normal file
View File

@@ -0,0 +1,35 @@
{
"network": [
[
"probeNetwork - default - end",
0,
0
],
[
"probeNetwork - default - start",
0,
0
]
],
"gfx": [
[
"probeGFX - default - end",
2,
4,
9,
1,
[
36864,
36864
],
[
202,
202
],
[
704,
704
]
]
]
}

35
render-test/metrics/android/render-tests/line-dasharray/round/zero-gap-width/metrics.json Normal file
View File

@@ -0,0 +1,35 @@
{
"network": [
[
"probeNetwork - default - end",
0,
0
],
[
"probeNetwork - default - start",
0,
0
]
],
"gfx": [
[
"probeGFX - default - end",
2,
4,
9,
1,
[
36864,
36864
],
[
202,
202
],
[
704,
704
]
]
]
}

35
render-test/metrics/android/render-tests/text-field/formatted-images-constant-size/metrics.json Normal file
View File

@@ -0,0 +1,35 @@
{
"network": [
[
"probeNetwork - default - end",
3,
161976
],
[
"probeNetwork - default - start",
0,
0
]
],
"gfx": [
[
"probeGFX - default - end",
2,
4,
13,
1,
[
24580,
24580
],
[
190,
190
],
[
1856,
1856
]
]
]
}

35
render-test/metrics/android/render-tests/text-field/formatted-images-line/metrics.json Normal file
View File

@@ -0,0 +1,35 @@
{
"network": [
[
"probeNetwork - default - end",
3,
120865
],
[
"probeNetwork - default - start",
0,
0
]
],
"gfx": [
[
"probeGFX - default - end",
3,
4,
11,
1,
[
28318,
28318
],
[
190,
190
],
[
1424,
1424
]
]
]
}

35
render-test/metrics/android/render-tests/text-field/formatted-images-multiline/metrics.json Normal file
View File

@@ -0,0 +1,35 @@
{
"network": [
[
"probeNetwork - default - end",
3,
120865
],
[
"probeNetwork - default - start",
0,
0
]
],
"gfx": [
[
"probeGFX - default - end",
2,
6,
13,
1,
[
44921,
44921
],
[
310,
310
],
[
3136,
3136
]
]
]
}

35
render-test/metrics/android/render-tests/text-field/formatted-images-variable-anchors-justification/metrics.json Normal file
View File

@@ -0,0 +1,35 @@
{
"network": [
[
"probeNetwork - default - end",
3,
161976
],
[
"probeNetwork - default - start",
0,
0
]
],
"gfx": [
[
"probeGFX - default - end",
1,
4,
9,
1,
[
24580,
24580
],
[
2290,
2290
],
[
24256,
24256
]
]
]
}

35
render-test/metrics/android/render-tests/text-field/formatted-images-vertical/metrics.json Normal file
View File

@@ -0,0 +1,35 @@
{
"network": [
[
"probeNetwork - default - end",
4,
255564
],
[
"probeNetwork - default - start",
0,
0
]
],
"gfx": [
[
"probeGFX - default - end",
1,
4,
9,
1,
[
35756,
35756
],
[
310,
310
],
[
3136,
3136
]
]
]
}

35
render-test/metrics/android/render-tests/text-field/formatted-images-zoom-dependent-size/metrics.json Normal file
View File

@@ -0,0 +1,35 @@
{
"network": [
[
"probeNetwork - default - end",
3,
161976
],
[
"probeNetwork - default - start",
0,
0
]
],
"gfx": [
[
"probeGFX - default - end",
2,
13,
13,
1,
[
98636,
98636
],
[
142,
142
],
[
1344,
1344
]
]
]
}

35
render-test/metrics/android/render-tests/text-field/formatted-images/metrics.json Normal file
View File

@@ -0,0 +1,35 @@
{
"network": [
[
"probeNetwork - default - end",
3,
161976
],
[
"probeNetwork - default - start",
0,
0
]
],
"gfx": [
[
"probeGFX - default - end",
3,
4,
17,
1,
[
52510,
52510
],
[
166,
166
],
[
1600,
1600
]
]
]
}