tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[test-runner] Add rebaseline flag

Browse Source 
Rebaseline flag allows to update or create new metric, in cases when
metric check is failed or expected metric is missing.
This commit is contained in:
Alexander Shalamov
2019-12-05 18:17:03 +02:00
parent 3d0001642b
commit a9ba572118
3 changed files with 24 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
render-test/runner.cpp
View File

@@ -316,11 +316,17 @@ bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage,
bool TestRunner::checkProbingResults(TestMetadata& metadata) {
if (metadata.metrics.isEmpty()) return true;
const auto writeMetrics = [&metadata](const mbgl::filesystem::path& path,
const std::string& message = std::string()) {
mbgl::filesystem::create_directories(path);
mbgl::util::write_file(path / "metrics.json", serializeMetrics(metadata.metrics));
metadata.errorMessage += message;
};
const std::vector<mbgl::filesystem::path>& expectedMetrics = metadata.paths.expectedMetrics;
if (updateResults == UpdateResults::METRICS) {
mbgl::filesystem::create_directories(expectedMetrics.back());
mbgl::util::write_file(expectedMetrics.back().string() + "/metrics.json", serializeMetrics(metadata.metrics));
return true;
writeMetrics(expectedMetrics.back(), " Updated expected metrics.");
return false;
}
// Check the possible paths in reverse order, so that the default path with the test style will only be checked in
@@ -347,8 +353,12 @@ bool TestRunner::checkProbingResults(TestMetadata& metadata) {
if (metadata.expectedMetrics.isEmpty()) {
metadata.errorMessage = "Failed to find metric expectations for: " + metadata.paths.stylePath.string();
if (updateResults == UpdateResults::REBASELINE) {
writeMetrics(expectedMetrics.back(), ". Created baseline for missing metrics.");
}
return false;
}
// Check file size metrics.
auto checkFileSize = [](TestMetadata& metadata) -> bool {
if (metadata.metrics.fileSize.empty()) return true;
@@ -567,8 +577,14 @@ bool TestRunner::checkProbingResults(TestMetadata& metadata) {
return true;
};
return checkFileSize(metadata) && checkMemory(metadata) && checkNetwork(metadata) && checkFps(metadata) &&
checkGfx(metadata);
bool checkResult = checkFileSize(metadata) && checkMemory(metadata) && checkNetwork(metadata) &&
checkFps(metadata) && checkGfx(metadata);
if (!checkResult && updateResults == UpdateResults::REBASELINE) {
writeMetrics(expectedMetrics.back(), " Rebaselined expected metric for failed test.");
}
return checkResult;
}
bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata, RunContext& ctx) {