tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[render-test] Fix metrics passing when failing

Browse Source 
Metrics will now fail, but the tool will return 0 when *ONLY* metrics
fail, to make the bot continue. We will also run ignored tests and
check if they are ignored but passing.
This commit is contained in:
Thiago Marcos P. Santos
2019-12-15 01:13:17 +02:00
parent 65140e3e60
commit 1c138124a6
3 changed files with 32 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
render-test/render_test.cpp
View File

@@ -119,6 +119,7 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
namespace mbgl {
int runRenderTests(int argc, char** argv, std::function<void()> testStatus) {
int returnCode = 0;
bool recycleMap;
bool shuffle;
uint32_t seed;
@@ -177,9 +178,11 @@ int runRenderTests(int argc, char** argv, std::function<void()> testStatus) {
}
}
bool errored = !metadata.errorMessage.empty() || !runner.run(metadata);
bool passed =
!errored && (!metadata.outputsImage || !metadata.diff.empty()) && metadata.difference <= metadata.allowed;
auto results = runner.run(metadata);
bool errored = !metadata.errorMessage.empty() || !(results.probeFailed || results.renderFailed);
bool imageOK = (!metadata.outputsImage || !metadata.diff.empty()) && metadata.difference <= metadata.allowed;
bool passed = !errored && imageOK;
if (shouldIgnore) {
if (passed) {
@@ -194,6 +197,14 @@ int runRenderTests(int argc, char** argv, std::function<void()> testStatus) {
printf(ANSI_COLOR_LIGHT_GRAY "* ignore %s (%s)" ANSI_COLOR_RESET "\n", id.c_str(), ignoreReason.c_str());
}
} else {
// Only fail the bots on render errors, this is a CI limitation that we need
// to succeed on metrics failed so the rebaseline bot can run next in the
// pipeline and collect the new baselines. The rebaseline bot will ultimately
// report the error and block the patch from being merged.
if (results.renderFailed == false || !imageOK) {
returnCode = 1;
}
if (passed) {
status = "passed";
color = "green";
@@ -246,7 +257,7 @@ int runRenderTests(int argc, char** argv, std::function<void()> testStatus) {
printf("Results at: %s\n", mbgl::filesystem::canonical(resultPath).c_str());
return stats.failedTests + stats.erroredTests == 0 ? 0 : 1;
return returnCode;
}
} // namespace mbgl