[build] Collect and publish new baselines

Also fail the bot in case there are new baselines.
This commit is contained in:
Thiago Marcos P. Santos
2019-12-12 22:18:47 +02:00
parent 0dbe80fe66
commit 8869d09b25
32 changed files with 163 additions and 153 deletions

4
.gitignore vendored
View File

@@ -6,7 +6,6 @@
*~
**/actual.png
**/diff.png
render-test/index.html
offline.db
new_offline.db
xcuserdata
@@ -38,6 +37,9 @@ test/fixtures/storage/assets.zip
buck-out
.buckd
# Generated metrics results
metrics/*.html
# Android RenderTestRunner
render-test/android/.*
render-test/android/app/build/

View File

@@ -6,35 +6,45 @@ workflows:
jobs:
- next-sanity-checks
- next-android-render-test-runner
- next-baselines:
requires:
- next-android-arm64-v8a-release
- next-android-armeabi-v7a-release
- next-android-render-test-runner
- next-android-x86-release
- next-android-x86_64-release
- next-linux-clang8-release
- next-linux-gcc8-release
- next-macos-xcode11-release
- next-build-template:
name: next-android-armeabi-v7a-release
executor_name: ubuntu-disco
target_is_android: true
config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_TOOLCHAIN_FILE=/opt/android/ndk-bundle/build/cmake/android.toolchain.cmake -DANDROID_CCACHE=/usr/bin/ccache -DANDROID_ABI=armeabi-v7a'
config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=/tmp/workspace/install/$CIRCLE_JOB -DCMAKE_TOOLCHAIN_FILE=/opt/android/ndk-bundle/build/cmake/android.toolchain.cmake -DANDROID_CCACHE=/usr/bin/ccache -DANDROID_ABI=armeabi-v7a'
install: true
- next-build-template:
name: next-android-arm64-v8a-release
executor_name: ubuntu-disco
target_is_android: true
config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_TOOLCHAIN_FILE=/opt/android/ndk-bundle/build/cmake/android.toolchain.cmake -DANDROID_CCACHE=/usr/bin/ccache -DANDROID_ABI=arm64-v8a'
config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=/tmp/workspace/install/$CIRCLE_JOB -DCMAKE_TOOLCHAIN_FILE=/opt/android/ndk-bundle/build/cmake/android.toolchain.cmake -DANDROID_CCACHE=/usr/bin/ccache -DANDROID_ABI=arm64-v8a'
install: true
- next-build-template:
name: next-android-x86-release
executor_name: ubuntu-disco
target_is_android: true
config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_TOOLCHAIN_FILE=/opt/android/ndk-bundle/build/cmake/android.toolchain.cmake -DANDROID_CCACHE=/usr/bin/ccache -DANDROID_ABI=x86'
config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=/tmp/workspace/install/$CIRCLE_JOB -DCMAKE_TOOLCHAIN_FILE=/opt/android/ndk-bundle/build/cmake/android.toolchain.cmake -DANDROID_CCACHE=/usr/bin/ccache -DANDROID_ABI=x86'
install: true
- next-build-template:
name: next-android-x86_64-release
executor_name: ubuntu-disco
target_is_android: true
config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_TOOLCHAIN_FILE=/opt/android/ndk-bundle/build/cmake/android.toolchain.cmake -DANDROID_CCACHE=/usr/bin/ccache -DANDROID_ABI=x86_64'
config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=/tmp/workspace/install/$CIRCLE_JOB -DCMAKE_TOOLCHAIN_FILE=/opt/android/ndk-bundle/build/cmake/android.toolchain.cmake -DANDROID_CCACHE=/usr/bin/ccache -DANDROID_ABI=x86_64'
install: true
- next-build-template:
name: next-linux-gcc8-release
executor_name: ubuntu-disco
target_is_linux: true
config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8'
config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=/tmp/workspace/install/$CIRCLE_JOB -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8'
install: true
metrics: true
style_tests: true
@@ -59,16 +69,8 @@ workflows:
name: next-linux-clang8-release
executor_name: ubuntu-disco
target_is_linux: true
requires:
- next-android-armeabi-v7a-release
- next-android-arm64-v8a-release
- next-android-x86-release
- next-android-x86_64-release
- next-linux-gcc8-release
- next-macos-xcode11-release
config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-8 -DCMAKE_CXX_COMPILER=clang++-8'
config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=/tmp/workspace/install/$CIRCLE_JOB -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-8 -DCMAKE_CXX_COMPILER=clang++-8'
install: true
binary_size: true
style_tests: true
- next-build-template:
name: next-FIXME-linux-asan
@@ -119,7 +121,7 @@ workflows:
name: next-macos-xcode11-release
executor_name: macos-11_0_0
target_is_macos: true
config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_CXX_COMPILER_LAUNCHER=ccache'
config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=/tmp/workspace/install/$CIRCLE_JOB -DCMAKE_CXX_COMPILER_LAUNCHER=ccache'
install: true
style_tests: true
- next-build-template:
@@ -268,10 +270,6 @@ commands:
name: Install
command: |
cmake --build build --target install/strip
- persist_to_workspace:
root: workspace
paths:
- '*'
next-test:
parameters:
test_params:
@@ -304,9 +302,13 @@ commands:
- ~/.ccache
- ~/.gradle
- run:
name: Collecting artifacts
name: Collecting results
when: always
command: |
mkdir -p /tmp/workspace/metrics && touch /tmp/workspace/.$CIRCLE_JOB
if [ -d metrics/$CIRCLE_JOB ]; then cp -r metrics/$CIRCLE_JOB /tmp/workspace/metrics; fi
mkdir -p /tmp/tests/baselines
if [ -f baselines.patch ]; then cp baselines.patch /tmp/tests/baselines; fi
mkdir -p /tmp/tests/nitpick
if [ -f nitpick.patch ]; then cp nitpick.patch /tmp/tests/nitpick; fi
mkdir -p /tmp/tests/render
@@ -319,6 +321,10 @@ commands:
if ls core* 1> /dev/null 2>&1; then cp core* /tmp/tests/coredumps; fi
mkdir -p /tmp/tests/valgrind
if ls build/Testing/Temporary/MemoryChecker.*.log 1> /dev/null 2>&1; then cp build/Testing/Temporary/MemoryChecker.*.log /tmp/tests/valgrind; fi
- persist_to_workspace:
root: /tmp/workspace
paths:
- '*'
- store_artifacts:
path: /tmp/tests
destination: tests
@@ -688,15 +694,14 @@ jobs:
if [[ -n "${GCLOUD_SERVICE_ACCOUNT_JSON}" && -z "${SKIP_FIREBASE:-}" ]]; then
testResult=$(gsutil ls gs://test-lab-186672a0qp5bq-ycr70axads3nc/render-test-app-${CIRCLE_BUILD_NUM}/**/*.html)
mkdir -p /tmp/tests/render-test-app
gsutil cp $testResult /tmp/tests/render-test-app | true
gsutil cp $testResult /tmp/tests/render-test-app
fi
- run:
name: Retrieve Rebaselined Metrics from gcloud
when: on_fail
command: |
if [[ -n "${GCLOUD_SERVICE_ACCOUNT_JSON}" && -z "${SKIP_FIREBASE:-}" ]]; then
metricsResult=$(gsutil ls -d gs://test-lab-186672a0qp5bq-ycr70axads3nc/render-test-app-${CIRCLE_BUILD_NUM}/*/*/sdcard/render-tests/)
gsutil -m cp -r $metricsResult metrics/next-android-render-test-runner/render-tests/ | true
metricsResult=$(gsutil ls -d gs://test-lab-186672a0qp5bq-ycr70axads3nc/render-test-app-${CIRCLE_BUILD_NUM}/*/*/sdcard/baselines/*) || true
gsutil -m cp -r $metricsResult metrics/next-android-render-test-runner/ || true
fi
- next-save
next-sanity-checks:
@@ -734,6 +739,24 @@ jobs:
run-clang-tidy-8 -quiet -j24 -p build $PWD/src/.*cpp $PWD/platform/.*cpp |tee clang-tidy.log
grep -o \\[[a-z].*-.*[a-z]\] clang-tidy.log |sort |uniq -c |sort -n
- next-save
next-baselines:
executor: ubuntu-disco
steps:
- checkout
- next-prepare
- next-prepare-linux
- attach_workspace:
at: /tmp/attach
- run:
name: 'Binary Size'
command: |
/tmp/attach/install/next-linux-gcc8-release/bin/mbgl-render-test-runner -u rebaseline -p metrics/next-binary-size.json
- run:
name: 'Matrics Baselines'
command: |
cp -r /tmp/attach/metrics .
git add -A && git diff --staged --exit-code | tee baselines.patch
- next-save
next-build-template:
parameters:
config_params:
@@ -759,9 +782,6 @@ jobs:
install:
type: boolean
default: false
binary_size:
type: boolean
default: false
metrics:
type: boolean
default: false
@@ -815,14 +835,6 @@ jobs:
- next-metrics:
step_name: 'Metrics'
metrics_params: '-u rebaseline -p metrics/$CIRCLE_JOB-metrics.json'
- when:
condition: << parameters.binary_size >>
steps:
- attach_workspace:
at: workspace
- next-metrics:
step_name: 'Binary Size'
metrics_params: '-u rebaseline -p metrics/binary-size.json'
- next-save
# ------------------------------------------------------------------------------
node-linux-release:

View File

@@ -1,3 +0,0 @@
{
"base_test_path":"tests/binary-size"
}

View File

@@ -1,12 +1,4 @@
{
"probes/memory/pass-memory-size-is-same": "TODO: Check with Mikhail why is this failing",
"binary-size/android-arm64-v8a": "skip - only makes sense for the binary size bot.",
"binary-size/android-armeabi-v7a": "skip - only makes sense for the binary size bot.",
"binary-size/android-x86": "skip - only makes sense for the binary size bot.",
"binary-size/android-x86_64": "skip - only makes sense for the binary size bot.",
"binary-size/linux-clang8": "skip - only makes sense for the binary size bot.",
"binary-size/linux-gcc8": "skip - only makes sense for the binary size bot.",
"binary-size/macos-xcode11": "skip - only makes sense for the binary size bot.",
"expression-tests/collator/accent-equals-de": "Locale-specific behavior changes based on platform.",
"expression-tests/in/assert-array": "https://github.com/mapbox/mapbox-gl-native/issues/15893",
"expression-tests/in/assert-string": "https://github.com/mapbox/mapbox-gl-native/issues/15893",
@@ -43,6 +35,7 @@
"probes/gfx/fail-vb-mem-mismatch": "Should fail, combined byte size of index buffers doesn't match the expectation.",
"probes/memory/fail-memory-size-is-too-big": "Should fail, memory size is bigger than expected.",
"probes/memory/fail-memory-size-is-too-small": "Should fail, memory size is smaller than expected.",
"probes/memory/pass-memory-size-is-same": "TODO: Check with Mikhail why is this failing",
"probes/network/fail-requests": "Should fail, number of requests higher than expected.",
"probes/network/fail-requests-transferred": "Should fail, number of requests higher than expected and amount of transferred data less than expected.",
"probes/network/fail-transferred": "Should fail, amount of transferred data higher than expected.",

View File

@@ -0,0 +1,4 @@
{
"base_test_path": "next-binary-size",
"metric_path": "next-binary-size"
}

View File

Before

Width:  |  Height:  |  Size: 225 B

After

Width:  |  Height:  |  Size: 225 B

View File

@@ -2,7 +2,7 @@
"file-size": [
[
"android-arm64-v8a",
"/src/workspace/next-android-arm64-v8a-release/lib/libmapbox-gl.so",
"/tmp/attach/install/next-android-arm64-v8a-release/lib/libmapbox-gl.so",
1907875
]
]

View File

@@ -3,7 +3,7 @@
"metadata": {
"test": {
"operations": [
["probeFileSize", "android-armeabi-v7a", "/src/workspace/next-android-armeabi-v7a-release/lib/libmapbox-gl.so", 0.01, "compressed"]
["probeFileSize", "android-arm64-v8a", "/tmp/attach/install/next-android-arm64-v8a-release/lib/libmapbox-gl.so", 0.01, "compressed"]
],
"width": 64,
"height": 64

View File

Before

Width:  |  Height:  |  Size: 225 B

After

Width:  |  Height:  |  Size: 225 B

View File

@@ -2,7 +2,7 @@
"file-size": [
[
"android-armeabi-v7a",
"/src/workspace/next-android-armeabi-v7a-release/lib/libmapbox-gl.so",
"/tmp/attach/install/next-android-armeabi-v7a-release/lib/libmapbox-gl.so",
1607847
]
]

View File

@@ -3,7 +3,7 @@
"metadata": {
"test": {
"operations": [
["probeFileSize", "android-arm64-v8a", "/src/workspace/next-android-arm64-v8a-release/lib/libmapbox-gl.so", 0.01, "compressed"]
["probeFileSize", "android-armeabi-v7a", "/tmp/attach/install/next-android-armeabi-v7a-release/lib/libmapbox-gl.so", 0.01, "compressed"]
],
"width": 64,
"height": 64

View File

Before

Width:  |  Height:  |  Size: 225 B

After

Width:  |  Height:  |  Size: 225 B

View File

@@ -2,7 +2,7 @@
"file-size": [
[
"android-x86",
"/src/workspace/next-android-x86-release/lib/libmapbox-gl.so",
"/tmp/attach/install/next-android-x86-release/lib/libmapbox-gl.so",
1934186
]
]

View File

@@ -3,7 +3,7 @@
"metadata": {
"test": {
"operations": [
["probeFileSize", "android-x86", "/src/workspace/next-android-x86-release/lib/libmapbox-gl.so", 0.01, "compressed"]
["probeFileSize", "android-x86", "/tmp/attach/install/next-android-x86-release/lib/libmapbox-gl.so", 0.01, "compressed"]
],
"width": 64,
"height": 64

View File

Before

Width:  |  Height:  |  Size: 225 B

After

Width:  |  Height:  |  Size: 225 B

View File

@@ -2,7 +2,7 @@
"file-size": [
[
"android-x86_64",
"/src/workspace/next-android-x86_64-release/lib/libmapbox-gl.so",
"/tmp/attach/install/next-android-x86_64-release/lib/libmapbox-gl.so",
1946541
]
]

View File

@@ -3,7 +3,7 @@
"metadata": {
"test": {
"operations": [
["probeFileSize", "android-x86_64", "/src/workspace/next-android-x86_64-release/lib/libmapbox-gl.so", 0.01, "compressed"]
["probeFileSize", "android-x86_64", "/tmp/attach/install/next-android-x86_64-release/lib/libmapbox-gl.so", 0.01, "compressed"]
],
"width": 64,
"height": 64

View File

Before

Width:  |  Height:  |  Size: 225 B

After

Width:  |  Height:  |  Size: 225 B

View File

@@ -2,17 +2,17 @@
"file-size": [
[
"mbgl-glfw",
"/src/workspace/next-linux-clang8-release/bin/mbgl-glfw",
"/tmp/attach/install/next-linux-clang8-release/bin/mbgl-glfw",
6290568
],
[
"mbgl-offline",
"/src/workspace/next-linux-clang8-release/bin/mbgl-offline",
"/tmp/attach/install/next-linux-clang8-release/bin/mbgl-offline",
5597912
],
[
"mbgl-render",
"/src/workspace/next-linux-clang8-release/bin/mbgl-render",
"/tmp/attach/install/next-linux-clang8-release/bin/mbgl-render",
6220728
]
]

View File

@@ -0,0 +1,24 @@
{
"version": 8,
"metadata": {
"test": {
"operations": [
["probeFileSize", "mbgl-glfw", "/tmp/attach/install/next-linux-clang8-release/bin/mbgl-glfw", 0.01],
["probeFileSize", "mbgl-offline", "/tmp/attach/install/next-linux-clang8-release/bin/mbgl-offline", 0.01],
["probeFileSize", "mbgl-render", "/tmp/attach/install/next-linux-clang8-release/bin/mbgl-render", 0.01]
],
"width": 64,
"height": 64
}
},
"sources": {},
"layers": [
{
"id": "background",
"type": "background",
"paint": {
"background-color": "green"
}
}
]
}

View File

Before

Width:  |  Height:  |  Size: 225 B

After

Width:  |  Height:  |  Size: 225 B

View File

@@ -2,17 +2,17 @@
"file-size": [
[
"mbgl-glfw",
"/src/workspace/next-linux-gcc8-release/bin/mbgl-glfw",
"/tmp/attach/install/next-linux-gcc8-release/bin/mbgl-glfw",
7323976
],
[
"mbgl-offline",
"/src/workspace/next-linux-gcc8-release/bin/mbgl-offline",
"/tmp/attach/install/next-linux-gcc8-release/bin/mbgl-offline",
6422760
],
[
"mbgl-render",
"/src/workspace/next-linux-gcc8-release/bin/mbgl-render",
"/tmp/attach/install/next-linux-gcc8-release/bin/mbgl-render",
7143752
]
]

View File

@@ -0,0 +1,24 @@
{
"version": 8,
"metadata": {
"test": {
"operations": [
["probeFileSize", "mbgl-glfw", "/tmp/attach/install/next-linux-gcc8-release/bin/mbgl-glfw", 0.01],
["probeFileSize", "mbgl-offline", "/tmp/attach/install/next-linux-gcc8-release/bin/mbgl-offline", 0.01],
["probeFileSize", "mbgl-render", "/tmp/attach/install/next-linux-gcc8-release/bin/mbgl-render", 0.01]
],
"width": 64,
"height": 64
}
},
"sources": {},
"layers": [
{
"id": "background",
"type": "background",
"paint": {
"background-color": "green"
}
}
]
}

View File

Before

Width:  |  Height:  |  Size: 225 B

After

Width:  |  Height:  |  Size: 225 B

View File

@@ -0,0 +1,19 @@
{
"file-size": [
[
"mbgl-glfw",
"/tmp/attach/install/next-macos-xcode11-release/bin/mbgl-glfw",
5502420
],
[
"mbgl-offline",
"/tmp/attach/install/next-macos-xcode11-release/bin/mbgl-offline",
5334136
],
[
"mbgl-render",
"/tmp/attach/install/next-macos-xcode11-release/bin/mbgl-render",
5384420
]
]
}

View File

@@ -0,0 +1,24 @@
{
"version": 8,
"metadata": {
"test": {
"operations": [
["probeFileSize", "mbgl-glfw", "/tmp/attach/install/next-macos-xcode11-release/bin/mbgl-glfw", 0.01],
["probeFileSize", "mbgl-offline", "/tmp/attach/install/next-macos-xcode11-release/bin/mbgl-offline", 0.01],
["probeFileSize", "mbgl-render", "/tmp/attach/install/next-macos-xcode11-release/bin/mbgl-render", 0.01]
],
"width": 64,
"height": 64
}
},
"sources": {},
"layers": [
{
"id": "background",
"type": "background",
"paint": {
"background-color": "green"
}
}
]
}

View File

@@ -1,24 +0,0 @@
{
"version": 8,
"metadata": {
"test": {
"operations": [
["probeFileSize", "mbgl-glfw", "/src/workspace/next-linux-clang8-release/bin/mbgl-glfw", 0.01],
["probeFileSize", "mbgl-offline", "/src/workspace/next-linux-clang8-release/bin/mbgl-offline", 0.01],
["probeFileSize", "mbgl-render", "/src/workspace/next-linux-clang8-release/bin/mbgl-render", 0.01]
],
"width": 64,
"height": 64
}
},
"sources": {},
"layers": [
{
"id": "background",
"type": "background",
"paint": {
"background-color": "green"
}
}
]
}

View File

@@ -1,24 +0,0 @@
{
"version": 8,
"metadata": {
"test": {
"operations": [
["probeFileSize", "mbgl-glfw", "/src/workspace/next-linux-gcc8-release/bin/mbgl-glfw", 0.01],
["probeFileSize", "mbgl-offline", "/src/workspace/next-linux-gcc8-release/bin/mbgl-offline", 0.01],
["probeFileSize", "mbgl-render", "/src/workspace/next-linux-gcc8-release/bin/mbgl-render", 0.01]
],
"width": 64,
"height": 64
}
},
"sources": {},
"layers": [
{
"id": "background",
"type": "background",
"paint": {
"background-color": "green"
}
}
]
}

View File

@@ -1,19 +0,0 @@
{
"file-size": [
[
"mbgl-glfw",
"/src/workspace/next-macos-xcode11-release/bin/mbgl-glfw",
5502420
],
[
"mbgl-offline",
"/src/workspace/next-macos-xcode11-release/bin/mbgl-offline",
5334136
],
[
"mbgl-render",
"/src/workspace/next-macos-xcode11-release/bin/mbgl-render",
5384420
]
]
}

View File

@@ -1,24 +0,0 @@
{
"version": 8,
"metadata": {
"test": {
"operations": [
["probeFileSize", "mbgl-glfw", "/src/workspace/next-macos-xcode11-release/bin/mbgl-glfw", 0.01],
["probeFileSize", "mbgl-offline", "/src/workspace/next-macos-xcode11-release/bin/mbgl-offline", 0.01],
["probeFileSize", "mbgl-render", "/src/workspace/next-macos-xcode11-release/bin/mbgl-render", 0.01]
],
"width": 64,
"height": 64
}
},
"sources": {},
"layers": [
{
"id": "background",
"type": "background",
"paint": {
"background-color": "green"
}
}
]
}

View File

@@ -121,3 +121,5 @@ target_link_libraries(
add_test(NAME mbgl-benchmark-runner COMMAND mbgl-benchmark-runner WORKING_DIRECTORY ${MBGL_ROOT})
add_test(NAME mbgl-test-runner COMMAND mbgl-test-runner WORKING_DIRECTORY ${MBGL_ROOT})
install(TARGETS mbgl-render-test-runner RUNTIME DESTINATION bin)

View File

@@ -467,7 +467,7 @@ mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& manife
std::vector<mbgl::filesystem::path> expectedMetricPaths{expectedMetricPath};
#if defined(__ANDROID__)
expectedMetricPaths.emplace_back("/sdcard/");
expectedMetricPaths.emplace_back("/sdcard/baselines/");
#endif
testPaths.emplace_back(testPath,
getTestExpectations(defaultExpectationPath, testId, expectationPaths),