tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[test runner] Test metrics reports. Check metrics results.

Browse Source
This commit is contained in:
Mikhail Pozdnyakov
2019-08-26 18:30:22 +03:00
parent d0a84cb7d9
commit 1b020eeb11
6 changed files with 128 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
render-test/main.cpp
View File

@@ -125,7 +125,7 @@ int main(int argc, char** argv) {
bool errored = !metadata.errorMessage.empty();
if (!errored) {
errored = runner.run(metadata) && !metadata.errorMessage.empty();
errored = !runner.run(metadata) || !metadata.errorMessage.empty();
}
bool passed = !errored && !metadata.diff.empty() && metadata.difference <= metadata.allowed;

11
render-test/metadata.hpp
View File

@@ -39,6 +39,12 @@ struct MemoryProbe {
size_t allocations;
};
class TestMetrics {
public:
bool isEmpty() const { return memory.empty(); }
std::map<std::string, MemoryProbe> memory;
};
struct TestMetadata {
TestMetadata() = default;
@@ -72,5 +78,6 @@ struct TestMetadata {
std::string errorMessage;
double difference = 0.0;
std::map<std::string, MemoryProbe> memoryProbes;
};
TestMetrics metrics;
TestMetrics expectedMetrics;
};

53
render-test/parser.cpp
View File

@@ -187,6 +187,29 @@ std::string serializeJsonValue(const mbgl::JSValue& value) {
return buffer.GetString();
}
std::string serializeMetrics(const TestMetrics& metrics) {
rapidjson::StringBuffer s;
rapidjson::Writer<rapidjson::StringBuffer> writer(s);
writer.StartObject();
// Start memory section.
writer.Key("memory");
writer.StartArray();
for (const auto& memoryProbe : metrics.memory) {
assert(!memoryProbe.first.empty());
writer.StartArray();
writer.String(memoryProbe.first.c_str());
writer.Uint64(memoryProbe.second.size);
writer.Uint64(memoryProbe.second.allocations);
writer.EndArray();
}
// End memory section.
writer.EndArray();
writer.EndObject();
return s.GetString();
}
std::vector<std::string> readExpectedEntries(const mbgl::filesystem::path& base) {
static const std::regex regex(".*expected.*.png");
@@ -290,6 +313,36 @@ std::vector<std::pair<std::string, std::string>> parseIgnores() {
return ignores;
}
TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
TestMetrics result;
auto maybeJson = readJson(path.string());
if (!maybeJson.is<mbgl::JSDocument>()) { // NOLINT
return result;
}
const auto& document = maybeJson.get<mbgl::JSDocument>();
if (document.HasMember("memory")) {
const mbgl::JSValue& memoryValue = document["memory"];
assert(memoryValue.IsArray());
for (auto& probeValue : memoryValue.GetArray()) {
assert(probeValue.IsArray());
assert(probeValue.Size() >= 3u);
assert(probeValue[0].IsString());
assert(probeValue[1].IsNumber());
assert(probeValue[2].IsNumber());
const std::string mark { probeValue[0].GetString(), probeValue[0].GetStringLength() };
assert(!mark.empty());
result.memory.emplace(std::piecewise_construct,
std::forward_as_tuple(std::move(mark)),
std::forward_as_tuple(probeValue[1].GetUint64(), probeValue[2].GetUint64()));
}
}
return result;
}
TestMetadata parseTestMetadata(const TestPaths& paths) {
TestMetadata metadata;
metadata.paths = paths;

11
render-test/parser.hpp
View File

@@ -1,5 +1,7 @@
#pragma once
#include "metadata.hpp"
#include <mbgl/util/rapidjson.hpp>
#include <mbgl/util/variant.hpp>
@@ -7,12 +9,6 @@
#include <string>
#include <vector>
#include "filesystem.hpp"
struct TestMetadata;
struct TestStatistics;
struct TestPaths;
using ErrorMessage = std::string;
using JSONReply = mbgl::variant<mbgl::JSDocument, ErrorMessage>;
@@ -20,9 +16,12 @@ using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::vector
JSONReply readJson(const mbgl::filesystem::path&);
std::string serializeJsonValue(const mbgl::JSValue&);
std::string serializeMetrics(const TestMetrics&);
std::vector<std::string> readExpectedEntries(const mbgl::filesystem::path& base);
TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path);
ArgumentsTuple parseArguments(int argc, char** argv);
std::vector<std::pair<std::string, std::string>> parseIgnores();

80
render-test/runner.cpp
View File

@@ -27,6 +27,7 @@
#include <cassert>
#include <regex>
#include <utility>
#include <sstream>
// static
const std::string& TestRunner::getBasePath() {
@@ -44,13 +45,13 @@ const std::vector<std::string>& TestRunner::getPlatformExpectationsPaths() {
return result;
}
bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& metadata) {
bool TestRunner::checkResults(mbgl::PremultipliedImage&& actualImage, TestMetadata& metadata) {
const std::string& base = metadata.paths.defaultExpectations();
const std::vector<mbgl::filesystem::path>& expectations = metadata.paths.expectations;
metadata.actual = mbgl::encodePNG(actual);
metadata.actual = mbgl::encodePNG(actualImage);
if (actual.size.isEmpty()) {
if (actualImage.size.isEmpty()) {
metadata.errorMessage = "Invalid size for actual image";
return false;
}
@@ -58,34 +59,46 @@ bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& met
#if !TEST_READ_ONLY
if (getenv("UPDATE_PLATFORM")) {
mbgl::filesystem::create_directories(expectations.back());
mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actual));
mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actualImage));
return true;
} else if (getenv("UPDATE_DEFAULT")) {
mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actual));
mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actualImage));
return true;
} else if (getenv("UPDATE_METRICS")) {
if (!metadata.metrics.isEmpty()) {
mbgl::filesystem::create_directories(expectations.back());
mbgl::util::write_file(expectations.back().string() + "/metrics.json", serializeMetrics(metadata.metrics));
return true;
}
}
mbgl::util::write_file(base + "/actual.png", metadata.actual);
#endif
mbgl::PremultipliedImage expected { actual.size };
mbgl::PremultipliedImage diff { actual.size };
mbgl::PremultipliedImage expectedImage { actualImage.size };
mbgl::PremultipliedImage imageDiff { actualImage.size };
double pixels = 0.0;
mbgl::filesystem::path expectedPath;
std::vector<std::string> expectedImagesPaths;
mbgl::filesystem::path expectedMetricsPath;
for (auto rit = expectations.rbegin(); rit!= expectations.rend(); ++rit) {
if (mbgl::filesystem::exists(*rit)) {
expectedPath = *rit;
break;
if (metadata.expectedMetrics.isEmpty()) {
mbgl::filesystem::path maybeExpectedMetricsPath{ *rit };
maybeExpectedMetricsPath.replace_filename("metrics.json");
metadata.expectedMetrics = readExpectedMetrics(maybeExpectedMetricsPath);
}
expectedImagesPaths = readExpectedEntries(*rit);
if (!expectedImagesPaths.empty()) break;
}
}
if (expectedPath.empty()) {
if (expectedImagesPaths.empty()) {
metadata.errorMessage = "Failed to find expectations for: " + metadata.paths.stylePath.string();
return false;
}
for (const auto& entry: readExpectedEntries(expectedPath)) {
for (const auto& entry: expectedImagesPaths) {
mbgl::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry);
if (!maybeExpectedImage) {
metadata.errorMessage = "Failed to load expected image " + entry;
@@ -94,23 +107,48 @@ bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& met
metadata.expected = *maybeExpectedImage;
expected = mbgl::decodeImage(*maybeExpectedImage);
expectedImage = mbgl::decodeImage(*maybeExpectedImage);
pixels = // implicitly converting from uint64_t
mapbox::pixelmatch(actual.data.get(), expected.data.get(), expected.size.width,
expected.size.height, diff.data.get(), 0.1285); // Defined in GL JS
mapbox::pixelmatch(actualImage.data.get(), expectedImage.data.get(), expectedImage.size.width,
expectedImage.size.height, imageDiff.data.get(), 0.1285); // Defined in GL JS
metadata.diff = mbgl::encodePNG(diff);
metadata.diff = mbgl::encodePNG(imageDiff);
#if !TEST_READ_ONLY
mbgl::util::write_file(base + "/diff.png", metadata.diff);
#endif
metadata.difference = pixels / expected.size.area();
metadata.difference = pixels / expectedImage.size.area();
if (metadata.difference <= metadata.allowed) {
break;
}
}
// Check memory metrics.
for (const auto& expected : metadata.expectedMetrics.memory) {
auto actual = metadata.metrics.memory.find(expected.first);
if (actual == metadata.metrics.memory.end()) {
metadata.errorMessage = "Failed to find memory probe: " + expected.first;
return false;
}
if (actual->second.size > expected.second.size) {
std::stringstream ss;
ss << "Allocated size at memory probe \" " << expected.first << "\" is "
<< actual->second.size << " bytes, expected is " << expected.second.size << " bytes.";
metadata.errorMessage = ss.str();
return false;
}
if (actual->second.allocations > expected.second.allocations) {
std::stringstream ss;
ss << "Number of allocations at memory probe \" " << expected.first << "\" is "
<< actual->second.allocations << ", expected is " << expected.second.allocations << ".";
metadata.errorMessage = ss.str();
return false;
}
}
return true;
}
@@ -407,9 +445,9 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
assert(operationArray[1].IsString());
std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength());
metadata.memoryProbes.emplace(std::piecewise_construct,
std::forward_as_tuple(std::move(mark)),
std::forward_as_tuple(AllocationIndex::getAllocatedSize(), AllocationIndex::getAllocationsCount()));
metadata.metrics.memory.emplace(std::piecewise_construct,
std::forward_as_tuple(std::move(mark)),
std::forward_as_tuple(AllocationIndex::getAllocatedSize(), AllocationIndex::getAllocationsCount()));
// probeMemoryEnd
} else if (operationArray[0].GetString() == memoryProbeEndOp) {
assert(AllocationIndex::isActive());
@@ -469,7 +507,7 @@ bool TestRunner::run(TestMetadata& metadata) {
return false;
}
return checkImage(std::move(image), metadata);
return checkResults(std::move(image), metadata);
}
void TestRunner::reset() {

2
render-test/runner.hpp
View File

@@ -21,7 +21,7 @@ public:
private:
bool runOperations(const std::string& key, TestMetadata&);
bool checkImage(mbgl::PremultipliedImage&& image, TestMetadata&);
bool checkResults(mbgl::PremultipliedImage&& image, TestMetadata&);
struct Impl {
Impl(const TestMetadata&);