953 lines
40 KiB
C++
953 lines
40 KiB
C++
#include <mbgl/map/camera.hpp>
|
|
#include <mbgl/map/map_observer.hpp>
|
|
#include <mbgl/renderer/renderer.hpp>
|
|
#include <mbgl/renderer/renderer_observer.hpp>
|
|
#include <mbgl/storage/file_source_manager.hpp>
|
|
#include <mbgl/style/image.hpp>
|
|
#include <mbgl/style/layer.hpp>
|
|
#include <mbgl/style/light.hpp>
|
|
#include <mbgl/style/rapidjson_conversion.hpp>
|
|
#include <mbgl/style/style.hpp>
|
|
#include <mbgl/util/chrono.hpp>
|
|
#include <mbgl/util/image.hpp>
|
|
#include <mbgl/util/io.hpp>
|
|
#include <mbgl/util/logging.hpp>
|
|
#include <mbgl/util/projection.hpp>
|
|
#include <mbgl/util/run_loop.hpp>
|
|
#include <mbgl/util/string.hpp>
|
|
#include <mbgl/util/tile_cover.hpp>
|
|
|
|
#include <mapbox/pixelmatch.hpp>
|
|
|
|
#include "test_runner_common.hpp"
|
|
#include "allocation_index.hpp"
|
|
#include "file_source.hpp"
|
|
#include "metadata.hpp"
|
|
#include "parser.hpp"
|
|
#include "runner.hpp"
|
|
|
|
#include <algorithm>
|
|
#include <cassert>
|
|
#include <utility>
|
|
#include <sstream>
|
|
|
|
#include <codecvt>
|
|
#include <locale>
|
|
|
|
using namespace mbgl;
|
|
using namespace TestOperationNames;
|
|
|
|
GfxProbe::GfxProbe(const mbgl::gfx::RenderingStats& stats, const GfxProbe& prev)
|
|
: numBuffers(stats.numBuffers),
|
|
numDrawCalls(stats.numDrawCalls),
|
|
numFrameBuffers(stats.numFrameBuffers),
|
|
numTextures(stats.numActiveTextures),
|
|
memIndexBuffers(stats.memIndexBuffers, std::max(stats.memIndexBuffers, prev.memIndexBuffers.peak)),
|
|
memVertexBuffers(stats.memVertexBuffers, std::max(stats.memVertexBuffers, prev.memVertexBuffers.peak)),
|
|
memTextures(stats.memTextures, std::max(stats.memTextures, prev.memTextures.peak)) {}
|
|
|
|
// static
|
|
gfx::HeadlessBackend::SwapBehaviour swapBehavior(MapMode mode) {
|
|
return mode == MapMode::Continuous ? gfx::HeadlessBackend::SwapBehaviour::Flush
|
|
: gfx::HeadlessBackend::SwapBehaviour::NoFlush;
|
|
}
|
|
|
|
std::string simpleDiff(const Value& result, const Value& expected) {
|
|
std::vector<std::string> resultTokens{tokenize(toJSON(result, 2, false))};
|
|
std::vector<std::string> expectedTokens{tokenize(toJSON(expected, 2, false))};
|
|
std::size_t maxLength = std::max(resultTokens.size(), expectedTokens.size());
|
|
std::ostringstream diff;
|
|
|
|
diff << "<pre>" << std::endl;
|
|
const auto flush =
|
|
[](const std::vector<std::string>& vec, std::size_t pos, std::ostringstream& out, std::string separator) {
|
|
for (std::size_t j = pos; j < vec.size(); ++j) {
|
|
out << separator << vec[j] << std::endl;
|
|
}
|
|
};
|
|
|
|
for (std::size_t i = 0; i < maxLength; ++i) {
|
|
if (resultTokens.size() <= i) {
|
|
flush(expectedTokens, i, diff, "-");
|
|
break;
|
|
}
|
|
|
|
if (expectedTokens.size() <= i) {
|
|
flush(resultTokens, i, diff, "+");
|
|
break;
|
|
}
|
|
|
|
if (resultTokens[i] != expectedTokens[i]) {
|
|
diff << "<b>"
|
|
<< "-" << expectedTokens[i] << "</b>" << std::endl;
|
|
diff << "<b>"
|
|
<< "+" << resultTokens[i] << "</b>" << std::endl;
|
|
} else {
|
|
diff << resultTokens[i] << std::endl;
|
|
}
|
|
}
|
|
diff << "</pre>" << std::endl;
|
|
return diff.str();
|
|
}
|
|
|
|
TestRunner::TestRunner(Manifest manifest_, UpdateResults updateResults_)
|
|
: manifest(std::move(manifest_)),
|
|
updateResults(updateResults_) {
|
|
registerProxyFileSource();
|
|
}
|
|
|
|
void TestRunner::registerProxyFileSource() {
|
|
static std::once_flag registerProxyFlag;
|
|
std::call_once(registerProxyFlag, [] {
|
|
auto* fileSourceManager = mbgl::FileSourceManager::get();
|
|
|
|
auto resourceLoaderFactory = fileSourceManager->unRegisterFileSourceFactory(
|
|
mbgl::FileSourceType::ResourceLoader);
|
|
auto factory = [defaultFactory = std::move(resourceLoaderFactory)](const mbgl::ResourceOptions& resourceOptions,
|
|
const mbgl::ClientOptions& clientOptions) {
|
|
assert(defaultFactory);
|
|
std::shared_ptr<FileSource> fileSource = defaultFactory(resourceOptions, clientOptions);
|
|
return std::make_unique<ProxyFileSource>(std::move(fileSource), resourceOptions, clientOptions);
|
|
};
|
|
|
|
fileSourceManager->registerFileSourceFactory(mbgl::FileSourceType::ResourceLoader, std::move(factory));
|
|
});
|
|
}
|
|
|
|
const Manifest& TestRunner::getManifest() const {
|
|
return manifest;
|
|
}
|
|
|
|
void TestRunner::doShuffle(uint32_t seed) {
|
|
manifest.doShuffle(seed);
|
|
}
|
|
|
|
void TestRunner::checkQueryTestResults(mbgl::PremultipliedImage&& actualImage,
|
|
std::vector<mbgl::Feature>&& features,
|
|
TestMetadata& metadata) {
|
|
const std::string& base = metadata.paths.defaultExpectations();
|
|
const std::vector<mbgl::filesystem::path>& expectations = metadata.paths.expectations;
|
|
|
|
metadata.actual = mbgl::encodePNG(actualImage);
|
|
|
|
if (actualImage.size.isEmpty()) {
|
|
metadata.errorMessage = "Invalid size for actual image";
|
|
metadata.renderErrored++;
|
|
return;
|
|
}
|
|
|
|
metadata.actualJson = toJSON(features, 2, false);
|
|
|
|
if (metadata.actualJson.empty()) {
|
|
metadata.errorMessage = "Invalid size for actual JSON";
|
|
metadata.renderErrored++;
|
|
return;
|
|
}
|
|
|
|
if (updateResults == UpdateResults::PLATFORM) {
|
|
mbgl::filesystem::create_directories(expectations.back());
|
|
mbgl::util::write_file(expectations.back().string() + "/expected.json", metadata.actualJson);
|
|
metadata.renderErrored++;
|
|
return;
|
|
} else if (updateResults == UpdateResults::DEFAULT) {
|
|
mbgl::util::write_file(base + "/expected.json", metadata.actualJson);
|
|
metadata.renderErrored++;
|
|
return;
|
|
}
|
|
|
|
mbgl::util::write_file(base + "/actual.json", metadata.actualJson);
|
|
|
|
std::vector<std::string> expectedJsonPaths;
|
|
mbgl::filesystem::path expectedMetricsPath;
|
|
for (auto rit = expectations.rbegin(); rit != expectations.rend(); ++rit) {
|
|
if (mbgl::filesystem::exists(*rit)) {
|
|
expectedJsonPaths = readExpectedJSONEntries(*rit);
|
|
if (!expectedJsonPaths.empty()) break;
|
|
}
|
|
}
|
|
|
|
if (expectedJsonPaths.empty()) {
|
|
metadata.errorMessage = "Failed to find expectations for: " + metadata.paths.stylePath.string();
|
|
metadata.renderErrored++;
|
|
return;
|
|
}
|
|
|
|
for (const auto& entry : expectedJsonPaths) {
|
|
auto maybeExpectedJson = readJson(entry);
|
|
if (maybeExpectedJson.is<mbgl::JSDocument>()) {
|
|
auto& expected = maybeExpectedJson.get<mbgl::JSDocument>();
|
|
|
|
mbgl::JSDocument actual;
|
|
actual.Parse<0>(metadata.actualJson);
|
|
if (actual.HasParseError()) {
|
|
metadata.errorMessage = "Error parsing actual JSON for: " + metadata.paths.stylePath.string();
|
|
metadata.renderErrored++;
|
|
return;
|
|
}
|
|
|
|
auto actualVal = mapbox::geojson::convert<mapbox::geojson::value>(actual);
|
|
auto expectedVal = mapbox::geojson::convert<mapbox::geojson::value>(expected);
|
|
bool equal = deepEqual(actualVal, expectedVal);
|
|
|
|
metadata.difference = !equal;
|
|
if (equal) {
|
|
metadata.diff = "Match";
|
|
} else {
|
|
metadata.diff = simpleDiff(actualVal, expectedVal);
|
|
metadata.renderFailed++;
|
|
}
|
|
} else {
|
|
metadata.errorMessage = "Failed to load expected JSON " + entry;
|
|
metadata.renderErrored++;
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
void TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage, TestMetadata& metadata) {
|
|
const std::string& base = metadata.paths.defaultExpectations();
|
|
const std::vector<mbgl::filesystem::path>& expectations = metadata.paths.expectations;
|
|
|
|
if (metadata.outputsImage) {
|
|
metadata.actual = mbgl::encodePNG(actualImage);
|
|
|
|
if (actualImage.size.isEmpty()) {
|
|
metadata.errorMessage = "Invalid size for actual image";
|
|
metadata.renderErrored++;
|
|
return;
|
|
}
|
|
|
|
if (updateResults == UpdateResults::PLATFORM) {
|
|
mbgl::filesystem::create_directories(expectations.back());
|
|
mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actualImage));
|
|
metadata.renderErrored++;
|
|
return;
|
|
} else if (updateResults == UpdateResults::DEFAULT) {
|
|
mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actualImage));
|
|
metadata.renderErrored++;
|
|
return;
|
|
}
|
|
|
|
mbgl::util::write_file(base + "/actual.png", metadata.actual);
|
|
|
|
mbgl::PremultipliedImage expectedImage{actualImage.size};
|
|
mbgl::PremultipliedImage imageDiff{actualImage.size};
|
|
|
|
double pixels = 0.0;
|
|
std::vector<std::string> expectedImagesPaths;
|
|
for (auto rit = expectations.rbegin(); rit != expectations.rend(); ++rit) {
|
|
if (mbgl::filesystem::exists(*rit)) {
|
|
expectedImagesPaths = readExpectedImageEntries(*rit);
|
|
if (!expectedImagesPaths.empty()) break;
|
|
}
|
|
}
|
|
|
|
if (expectedImagesPaths.empty()) {
|
|
metadata.errorMessage = "Failed to find expectations for: " + metadata.paths.stylePath.string();
|
|
metadata.renderErrored++;
|
|
return;
|
|
}
|
|
|
|
for (const auto& entry : expectedImagesPaths) {
|
|
std::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry);
|
|
if (!maybeExpectedImage) {
|
|
metadata.errorMessage = "Failed to load expected image " + entry;
|
|
metadata.renderErrored++;
|
|
return;
|
|
}
|
|
|
|
metadata.expected = *maybeExpectedImage;
|
|
|
|
expectedImage = mbgl::decodeImage(*maybeExpectedImage);
|
|
|
|
pixels = static_cast<double>(mapbox::pixelmatch(actualImage.data.get(),
|
|
expectedImage.data.get(),
|
|
expectedImage.size.width,
|
|
expectedImage.size.height,
|
|
imageDiff.data.get(),
|
|
0.1285) // Defined in GL JS
|
|
);
|
|
|
|
metadata.diff = mbgl::encodePNG(imageDiff);
|
|
|
|
mbgl::util::write_file(base + "/diff.png", metadata.diff);
|
|
|
|
metadata.difference = pixels / expectedImage.size.area();
|
|
if (metadata.difference <= metadata.allowed) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (metadata.difference > metadata.allowed) {
|
|
metadata.renderFailed++;
|
|
}
|
|
}
|
|
}
|
|
|
|
void TestRunner::checkProbingResults(TestMetadata& resultMetadata) {
|
|
if (resultMetadata.metrics.isEmpty()) return;
|
|
const auto writeMetrics = [&resultMetadata](const mbgl::filesystem::path& path,
|
|
const std::string& message = std::string()) {
|
|
try {
|
|
mbgl::filesystem::create_directories(path);
|
|
mbgl::util::write_file((path / "metrics.json").generic_string(), serializeMetrics(resultMetadata.metrics));
|
|
resultMetadata.errorMessage += message;
|
|
} catch (mbgl::filesystem::filesystem_error& ex) {
|
|
const auto msg = "Failed to write metrics. path='" + path.string() + "' err='" + ex.what() +
|
|
"' msg=" + message;
|
|
resultMetadata.errorMessage += msg;
|
|
Log::Warning(Event::General, msg);
|
|
}
|
|
};
|
|
|
|
const std::vector<mbgl::filesystem::path>& expectedMetrics = resultMetadata.paths.expectedMetrics;
|
|
if (updateResults == UpdateResults::METRICS) {
|
|
writeMetrics(expectedMetrics.back(), " Updated expected metrics.");
|
|
resultMetadata.metricsErrored++;
|
|
return;
|
|
}
|
|
|
|
// Check the possible paths in reverse order, so that the default path with
|
|
// the test style will only be checked in the very end.
|
|
std::vector<std::string> expectedMetricsPaths;
|
|
for (auto rit = expectedMetrics.rbegin(); rit != expectedMetrics.rend(); ++rit) {
|
|
if (mbgl::filesystem::exists(*rit)) {
|
|
expectedMetricsPaths = readExpectedMetricEntries(*rit);
|
|
if (!expectedMetricsPaths.empty()) break;
|
|
}
|
|
}
|
|
|
|
// In case no metrics.json is found, skip assigning the expectedMetrics to
|
|
// metadata, otherwise, take the first found metrics.
|
|
for (const auto& entry : expectedMetricsPaths) {
|
|
auto maybeExpectedMetrics = readExpectedMetrics(entry);
|
|
if (maybeExpectedMetrics.isEmpty()) {
|
|
resultMetadata.errorMessage = "Failed to load expected metrics " + entry;
|
|
resultMetadata.metricsErrored++;
|
|
return;
|
|
}
|
|
resultMetadata.expectedMetrics = maybeExpectedMetrics;
|
|
break;
|
|
}
|
|
|
|
if (resultMetadata.expectedMetrics.isEmpty()) {
|
|
resultMetadata.errorMessage = "Failed to find metric expectations for: " +
|
|
resultMetadata.paths.stylePath.string();
|
|
if (updateResults == UpdateResults::REBASELINE && !resultMetadata.ignoredTest) {
|
|
writeMetrics(expectedMetrics.back(), ". Created baseline for missing metrics.");
|
|
}
|
|
resultMetadata.metricsErrored++;
|
|
return;
|
|
}
|
|
|
|
// Check file size metrics.
|
|
auto checkFileSize = [](TestMetadata& metadata) {
|
|
if (metadata.metrics.fileSize.empty()) return;
|
|
for (const auto& expected : metadata.expectedMetrics.fileSize) {
|
|
auto actual = metadata.metrics.fileSize.find(expected.first);
|
|
if (actual == metadata.metrics.fileSize.end()) {
|
|
metadata.errorMessage = "Failed to find fileSize probe: " + expected.first;
|
|
metadata.metricsErrored++;
|
|
return;
|
|
}
|
|
if (actual->second.path != expected.second.path) {
|
|
std::stringstream ss;
|
|
ss << "Comparing different files at probe \"" << expected.first << "\": " << actual->second.path
|
|
<< ", expected is " << expected.second.path << ".";
|
|
metadata.errorMessage = ss.str();
|
|
metadata.metricsErrored++;
|
|
return;
|
|
}
|
|
|
|
auto result = checkValue(static_cast<float>(expected.second.size),
|
|
static_cast<float>(actual->second.size),
|
|
actual->second.tolerance);
|
|
if (!std::get<bool>(result)) {
|
|
std::stringstream ss;
|
|
ss << "File size does not match at probe \"" << expected.first << "\" for file \""
|
|
<< expected.second.path << "\": " << actual->second.size << ", expected is " << expected.second.size
|
|
<< ".";
|
|
|
|
metadata.errorMessage += metadata.errorMessage.empty() ? ss.str() : "\n" + ss.str();
|
|
metadata.metricsFailed++;
|
|
}
|
|
}
|
|
};
|
|
auto checkMemory = [](TestMetadata& metadata) {
|
|
if (metadata.metrics.memory.empty()) return;
|
|
#if !defined(SANITIZE)
|
|
// Check memory metrics.
|
|
for (const auto& expected : metadata.expectedMetrics.memory) {
|
|
auto actual = metadata.metrics.memory.find(expected.first);
|
|
if (actual == metadata.metrics.memory.end()) {
|
|
metadata.errorMessage = "Failed to find memory probe: " + expected.first;
|
|
metadata.metricsErrored++;
|
|
return;
|
|
}
|
|
bool passed{false};
|
|
float delta{0.0f};
|
|
std::stringstream errorStream;
|
|
std::tie(passed, delta) = MemoryProbe::checkPeak(expected.second, actual->second);
|
|
if (!passed) {
|
|
errorStream << "Allocated memory peak size at probe \"" << expected.first << "\" is "
|
|
<< actual->second.peak << " bytes, expected is " << expected.second.peak << "±" << delta
|
|
<< " bytes.";
|
|
|
|
metadata.metricsFailed++;
|
|
}
|
|
|
|
std::tie(passed, delta) = MemoryProbe::checkAllocations(expected.second, actual->second);
|
|
if (!passed) {
|
|
errorStream << "Number of allocations at probe \"" << expected.first << "\" is "
|
|
<< actual->second.allocations << ", expected is " << expected.second.allocations << "±"
|
|
<< std::round(delta) << " allocations.";
|
|
|
|
metadata.metricsFailed++;
|
|
}
|
|
|
|
metadata.errorMessage += metadata.errorMessage.empty() ? errorStream.str() : "\n" + errorStream.str();
|
|
}
|
|
#endif // !defined(SANITIZE)
|
|
};
|
|
|
|
// Check network metrics.
|
|
auto checkNetwork = [](TestMetadata& metadata) {
|
|
if (metadata.metrics.network.empty()) return;
|
|
#if !defined(SANITIZE)
|
|
for (const auto& expected : metadata.expectedMetrics.network) {
|
|
auto actual = metadata.metrics.network.find(expected.first);
|
|
if (actual == metadata.metrics.network.end()) {
|
|
metadata.errorMessage = "Failed to find network probe: " + expected.first;
|
|
metadata.metricsErrored++;
|
|
return;
|
|
}
|
|
std::stringstream ss;
|
|
if (actual->second.requests != expected.second.requests) {
|
|
ss << "Number of requests at probe \"" << expected.first << "\" is " << actual->second.requests
|
|
<< ", expected is " << expected.second.requests << ". ";
|
|
metadata.metricsFailed++;
|
|
}
|
|
if (actual->second.transferred != expected.second.transferred) {
|
|
ss << "Transferred data at probe \"" << expected.first << "\" is " << actual->second.transferred
|
|
<< " bytes, expected is " << expected.second.transferred << " bytes.";
|
|
metadata.metricsFailed++;
|
|
}
|
|
metadata.errorMessage += metadata.errorMessage.empty() ? ss.str() : "\n" + ss.str();
|
|
}
|
|
#endif // !defined(SANITIZE)
|
|
};
|
|
// Check fps metrics
|
|
auto checkFps = [](TestMetadata& metadata) {
|
|
if (metadata.metrics.fps.empty()) return;
|
|
for (const auto& expected : metadata.expectedMetrics.fps) {
|
|
auto actual = metadata.metrics.fps.find(expected.first);
|
|
if (actual == metadata.metrics.fps.end()) {
|
|
metadata.errorMessage = "Failed to find fps probe: " + expected.first;
|
|
metadata.metricsErrored++;
|
|
return;
|
|
}
|
|
auto result = checkValue(expected.second.average, actual->second.average, expected.second.tolerance);
|
|
std::stringstream ss;
|
|
if (!std::get<bool>(result)) {
|
|
ss << "Average fps at probe \"" << expected.first << "\" is " << actual->second.average
|
|
<< ", expected to be " << expected.second.average << " with tolerance of "
|
|
<< expected.second.tolerance;
|
|
metadata.metricsFailed++;
|
|
}
|
|
result = checkValue(expected.second.minOnePc, actual->second.minOnePc, expected.second.tolerance);
|
|
if (!std::get<bool>(result)) {
|
|
ss << "Minimum(1%) fps at probe \"" << expected.first << "\" is " << actual->second.minOnePc
|
|
<< ", expected to be " << expected.second.minOnePc << " with tolerance of "
|
|
<< expected.second.tolerance;
|
|
metadata.metricsFailed++;
|
|
}
|
|
metadata.errorMessage += metadata.errorMessage.empty() ? ss.str() : "\n" + ss.str();
|
|
}
|
|
};
|
|
// Check gfx metrics
|
|
auto checkGfx = [](TestMetadata& metadata) {
|
|
if (metadata.metrics.gfx.empty()) return;
|
|
for (const auto& expected : metadata.expectedMetrics.gfx) {
|
|
auto actual = metadata.metrics.gfx.find(expected.first);
|
|
if (actual == metadata.metrics.gfx.end()) {
|
|
metadata.errorMessage = "Failed to find gfx probe: " + expected.first;
|
|
metadata.metricsErrored++;
|
|
return;
|
|
}
|
|
|
|
const auto& probeName = expected.first;
|
|
const auto& expectedValue = expected.second;
|
|
const auto& actualValue = actual->second;
|
|
std::stringstream ss;
|
|
|
|
if (expectedValue.numDrawCalls != actualValue.numDrawCalls) {
|
|
if (!metadata.errorMessage.empty()) ss << std::endl;
|
|
ss << "Number of draw calls at probe\"" << probeName << "\" is " << actualValue.numDrawCalls
|
|
<< ", expected is " << expectedValue.numDrawCalls;
|
|
metadata.metricsFailed++;
|
|
}
|
|
|
|
if (expectedValue.numTextures != actualValue.numTextures) {
|
|
if (!metadata.errorMessage.empty()) ss << std::endl;
|
|
ss << "Number of textures at probe \"" << probeName << "\" is " << actualValue.numTextures
|
|
<< ", expected is " << expectedValue.numTextures;
|
|
metadata.metricsFailed++;
|
|
}
|
|
|
|
if (expectedValue.numBuffers != actualValue.numBuffers) {
|
|
if (!metadata.errorMessage.empty()) ss << std::endl;
|
|
ss << "Number of vertex and index buffers at probe \"" << probeName << "\" is "
|
|
<< actualValue.numBuffers << ", expected is " << expectedValue.numBuffers;
|
|
metadata.metricsFailed++;
|
|
}
|
|
|
|
if (expectedValue.numFrameBuffers != actualValue.numFrameBuffers) {
|
|
if (!metadata.errorMessage.empty()) ss << std::endl;
|
|
ss << "Number of frame buffers at probe \"" << probeName << "\" is " << actualValue.numFrameBuffers
|
|
<< ", expected is " << expectedValue.numFrameBuffers;
|
|
metadata.metricsFailed++;
|
|
}
|
|
|
|
if (expectedValue.memTextures.peak != actualValue.memTextures.peak) {
|
|
if (!metadata.errorMessage.empty()) ss << std::endl;
|
|
ss << "Allocated texture memory peak size at probe \"" << probeName << "\" is "
|
|
<< actualValue.memTextures.peak << " bytes, expected is " << expectedValue.memTextures.peak
|
|
<< " bytes";
|
|
metadata.metricsFailed++;
|
|
}
|
|
|
|
if (expectedValue.memIndexBuffers.peak != actualValue.memIndexBuffers.peak) {
|
|
if (!metadata.errorMessage.empty()) ss << std::endl;
|
|
ss << "Allocated index buffer memory peak size at probe \"" << probeName << "\" is "
|
|
<< actualValue.memIndexBuffers.peak << " bytes, expected is " << expectedValue.memIndexBuffers.peak
|
|
<< " bytes";
|
|
metadata.metricsFailed++;
|
|
}
|
|
|
|
if (expectedValue.memVertexBuffers.peak != actualValue.memVertexBuffers.peak) {
|
|
if (!metadata.errorMessage.empty()) ss << std::endl;
|
|
ss << "Allocated vertex buffer memory peak size at probe \"" << probeName << "\" is "
|
|
<< actualValue.memVertexBuffers.peak << " bytes, expected is " << expectedValue.memVertexBuffers.peak
|
|
<< " bytes";
|
|
metadata.metricsFailed++;
|
|
}
|
|
|
|
metadata.errorMessage += metadata.errorMessage.empty() ? ss.str() : "\n" + ss.str();
|
|
}
|
|
};
|
|
|
|
checkFileSize(resultMetadata);
|
|
checkMemory(resultMetadata);
|
|
checkNetwork(resultMetadata);
|
|
checkFps(resultMetadata);
|
|
checkGfx(resultMetadata);
|
|
|
|
if (resultMetadata.ignoredTest) {
|
|
return;
|
|
}
|
|
|
|
if ((resultMetadata.metricsErrored || resultMetadata.metricsFailed) && updateResults == UpdateResults::REBASELINE) {
|
|
writeMetrics(expectedMetrics.back(), " Rebaselined expected metric for failed test.");
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
|
|
TestOperation unsupportedOperation(const std::string& operation) {
|
|
return [operation](TestContext& ctx) {
|
|
ctx.getMetadata().errorMessage = std::string("Unsupported operation: ") + operation;
|
|
return false;
|
|
};
|
|
}
|
|
|
|
TestOperations getBeforeOperations(const Manifest& manifest) {
|
|
static const std::string mark = " - default - start";
|
|
TestOperations result;
|
|
for (const std::string& probe : manifest.getProbes()) {
|
|
if (memoryProbeOp == probe) {
|
|
result.emplace_back([](TestContext& ctx) {
|
|
assert(!AllocationIndex::isActive());
|
|
AllocationIndex::setActive(true);
|
|
ctx.getMetadata().metrics.memory.emplace(std::piecewise_construct,
|
|
std::forward_as_tuple(memoryProbeOp + mark),
|
|
std::forward_as_tuple(AllocationIndex::getAllocatedSizePeak(),
|
|
AllocationIndex::getAllocationsCount()));
|
|
return true;
|
|
});
|
|
continue;
|
|
}
|
|
if (gfxProbeOp == probe) {
|
|
result.emplace_back([](TestContext& ctx) {
|
|
assert(!ctx.gfxProbeActive);
|
|
ctx.gfxProbeActive = true;
|
|
ctx.baselineGfxProbe = ctx.activeGfxProbe;
|
|
return true;
|
|
});
|
|
continue;
|
|
}
|
|
|
|
if (networkProbeOp == probe) {
|
|
result.emplace_back([](TestContext& ctx) {
|
|
assert(!ProxyFileSource::isTrackingActive());
|
|
ProxyFileSource::setTrackingActive(true);
|
|
ctx.getMetadata().metrics.network.emplace(
|
|
std::piecewise_construct,
|
|
std::forward_as_tuple(networkProbeOp + mark),
|
|
std::forward_as_tuple(ProxyFileSource::getRequestCount(), ProxyFileSource::getTransferredSize()));
|
|
return true;
|
|
});
|
|
continue;
|
|
}
|
|
result.emplace_back(unsupportedOperation(probe));
|
|
}
|
|
return result;
|
|
}
|
|
|
|
TestOperations getAfterOperations(const Manifest& manifest) {
|
|
static const std::string mark = " - default - end";
|
|
TestOperations result;
|
|
for (const std::string& probe : manifest.getProbes()) {
|
|
if (memoryProbeOp == probe) {
|
|
result.emplace_back([](TestContext& ctx) {
|
|
assert(AllocationIndex::isActive());
|
|
auto emplaced = ctx.getMetadata().metrics.memory.emplace(
|
|
std::piecewise_construct,
|
|
std::forward_as_tuple(memoryProbeOp + mark),
|
|
std::forward_as_tuple(AllocationIndex::getAllocatedSizePeak(),
|
|
AllocationIndex::getAllocationsCount()));
|
|
assert(emplaced.second);
|
|
// TODO: Improve tolerance handling for memory tests.
|
|
emplaced.first->second.tolerance = 0.2f;
|
|
AllocationIndex::setActive(false);
|
|
AllocationIndex::reset();
|
|
return true;
|
|
});
|
|
continue;
|
|
}
|
|
if (gfxProbeOp == probe) {
|
|
result.emplace_back([](TestContext& ctx) {
|
|
// Compare memory allocations to the baseline probe
|
|
GfxProbe metricProbe = ctx.activeGfxProbe;
|
|
metricProbe.memIndexBuffers.peak -= ctx.baselineGfxProbe.memIndexBuffers.peak;
|
|
metricProbe.memVertexBuffers.peak -= ctx.baselineGfxProbe.memVertexBuffers.peak;
|
|
metricProbe.memTextures.peak -= ctx.baselineGfxProbe.memTextures.peak;
|
|
ctx.getMetadata().metrics.gfx.insert({gfxProbeOp + mark, metricProbe});
|
|
|
|
ctx.gfxProbeActive = false;
|
|
return true;
|
|
});
|
|
continue;
|
|
}
|
|
if (networkProbeOp == probe) {
|
|
result.emplace_back([](TestContext& ctx) {
|
|
ctx.getMetadata().metrics.network.emplace(
|
|
std::piecewise_construct,
|
|
std::forward_as_tuple(networkProbeOp + mark),
|
|
std::forward_as_tuple(ProxyFileSource::getRequestCount(), ProxyFileSource::getTransferredSize()));
|
|
ProxyFileSource::setTrackingActive(false);
|
|
return true;
|
|
});
|
|
continue;
|
|
}
|
|
result.emplace_back(unsupportedOperation(probe));
|
|
}
|
|
return result;
|
|
}
|
|
|
|
void resetContext(const TestMetadata& metadata, TestContext& ctx) {
|
|
ctx.getFrontend().getRenderer()->clearData();
|
|
ctx.getFrontend().setSize(metadata.size);
|
|
auto& map = ctx.getMap();
|
|
map.setSize(metadata.size);
|
|
map.setProjectionMode(mbgl::ProjectionMode()
|
|
.withAxonometric(metadata.axonometric)
|
|
.withXSkew(metadata.xSkew)
|
|
.withYSkew(metadata.ySkew));
|
|
map.setDebug(metadata.debug);
|
|
map.getStyle().loadJSON(serializeJsonValue(metadata.document));
|
|
}
|
|
|
|
LatLng getTileCenterCoordinates(const UnwrappedTileID& tileId) {
|
|
double scale = (1 << tileId.canonical.z);
|
|
Point<double> tileCenter{(tileId.canonical.x + 0.5) * util::tileSize_D,
|
|
(tileId.canonical.y + 0.5) * util::tileSize_D};
|
|
return Projection::unproject(tileCenter, scale);
|
|
}
|
|
|
|
uint32_t getTileScreenPixelSize(float pixelRatio) {
|
|
return static_cast<uint32_t>(util::tileSize_D * pixelRatio);
|
|
}
|
|
|
|
uint32_t getImageTileOffset(const std::set<uint32_t>& dims, uint32_t dim, float pixelRatio) {
|
|
auto it = dims.find(dim);
|
|
if (it == dims.end()) {
|
|
assert(false);
|
|
return 0;
|
|
}
|
|
return static_cast<uint32_t>(std::distance(dims.begin(), it)) * getTileScreenPixelSize(pixelRatio);
|
|
}
|
|
|
|
} // namespace
|
|
|
|
TestRunner::Impl::Impl(const TestMetadata& metadata,
|
|
const mbgl::ResourceOptions& resourceOptions,
|
|
const mbgl::ClientOptions& clientOptions)
|
|
: observer(std::make_unique<TestRunnerMapObserver>()),
|
|
frontend(metadata.size, metadata.pixelRatio, swapBehavior(metadata.mapMode)),
|
|
fileSource(mbgl::FileSourceManager::get()->getFileSource(
|
|
mbgl::FileSourceType::ResourceLoader, resourceOptions, clientOptions)),
|
|
map(frontend,
|
|
*observer,
|
|
mbgl::MapOptions()
|
|
.withMapMode(metadata.mapMode)
|
|
.withSize(metadata.size)
|
|
.withPixelRatio(metadata.pixelRatio)
|
|
.withCrossSourceCollisions(metadata.crossSourceCollisions),
|
|
resourceOptions,
|
|
clientOptions) {
|
|
if (metadata.maxPitch) {
|
|
map.setBounds(BoundOptions().withMaxPitch(*metadata.maxPitch));
|
|
}
|
|
}
|
|
|
|
TestRunner::Impl::~Impl() {}
|
|
|
|
void TestRunner::run(TestMetadata& metadata) {
|
|
AllocationIndex::setActive(false);
|
|
AllocationIndex::reset();
|
|
ProxyFileSource::setTrackingActive(false);
|
|
|
|
struct ContextImpl final : public TestContext {
|
|
ContextImpl(TestMetadata& metadata_)
|
|
: metadata(metadata_) {}
|
|
HeadlessFrontend& getFrontend() override {
|
|
assert(runnerImpl);
|
|
return runnerImpl->frontend;
|
|
}
|
|
Map& getMap() override {
|
|
assert(runnerImpl);
|
|
return runnerImpl->map;
|
|
}
|
|
|
|
FileSource& getFileSource() override {
|
|
assert(runnerImpl);
|
|
return *runnerImpl->fileSource;
|
|
}
|
|
|
|
TestRunnerMapObserver& getObserver() override {
|
|
assert(runnerImpl);
|
|
return *runnerImpl->observer;
|
|
}
|
|
TestMetadata& getMetadata() override { return metadata; }
|
|
|
|
TestRunner::Impl* runnerImpl = nullptr;
|
|
TestMetadata& metadata;
|
|
};
|
|
|
|
ContextImpl ctx(metadata);
|
|
|
|
if (!metadata.ignoredTest) {
|
|
for (const auto& operation : getBeforeOperations(manifest)) {
|
|
if (!operation(ctx)) return;
|
|
}
|
|
}
|
|
|
|
std::string key = mbgl::util::toString(uint32_t(metadata.mapMode)) + "/" +
|
|
mbgl::util::toString(metadata.pixelRatio) + "/" +
|
|
mbgl::util::toString(uint32_t(metadata.crossSourceCollisions));
|
|
|
|
if (maps.find(key) == maps.end()) {
|
|
maps[key] = std::make_unique<TestRunner::Impl>(
|
|
metadata,
|
|
mbgl::ResourceOptions().withCachePath(manifest.getCachePath()).withApiKey(manifest.getApiKey()),
|
|
mbgl::ClientOptions());
|
|
}
|
|
|
|
ctx.runnerImpl = maps[key].get();
|
|
auto& frontend = ctx.getFrontend();
|
|
auto& map = ctx.getMap();
|
|
|
|
resetContext(metadata, ctx);
|
|
auto camera = map.getStyle().getDefaultCamera();
|
|
|
|
HeadlessFrontend::RenderResult result{};
|
|
std::string cutOffLabelsReport;
|
|
std::string duplicationsReport;
|
|
|
|
if (metadata.mapMode == MapMode::Tile) {
|
|
assert(camera.zoom);
|
|
assert(camera.center);
|
|
auto tileIds = util::tileCover(map.latLngBoundsForCamera(camera), static_cast<uint8_t>(*camera.zoom));
|
|
assert(!tileIds.empty());
|
|
std::set<uint32_t> xDims;
|
|
std::set<uint32_t> yDims;
|
|
struct SymbolLocationAndStatus {
|
|
UnwrappedTileID tileId;
|
|
mapbox::geometry::box<float> location;
|
|
bool placed;
|
|
bool isIcon;
|
|
};
|
|
std::map<std::u16string, std::vector<SymbolLocationAndStatus>> symbolIndex;
|
|
|
|
for (const auto& tileId : tileIds) {
|
|
xDims.insert(tileId.canonical.x);
|
|
yDims.insert(tileId.canonical.y);
|
|
assert(tileId.canonical.z == uint8_t(*camera.zoom));
|
|
}
|
|
auto tileScreenSize = getTileScreenPixelSize(metadata.pixelRatio);
|
|
|
|
result.image = PremultipliedImage(
|
|
{uint32_t(xDims.size()) * tileScreenSize, uint32_t(yDims.size()) * tileScreenSize});
|
|
for (const auto& tileId : tileIds) {
|
|
resetContext(metadata, ctx);
|
|
ctx.getFrontend().getRenderer()->collectPlacedSymbolData(true);
|
|
auto cameraForTile{camera};
|
|
cameraForTile.withCenter(getTileCenterCoordinates(tileId));
|
|
map.jumpTo(cameraForTile);
|
|
|
|
auto resultForTile = runTest(metadata, ctx);
|
|
if (!resultForTile.image.valid()) {
|
|
metadata.errorMessage = "Failed rendering tile: " + util::toString(tileId);
|
|
return;
|
|
}
|
|
auto xOffset = getImageTileOffset(xDims, tileId.canonical.x, metadata.pixelRatio);
|
|
auto yOffset = getImageTileOffset(yDims, tileId.canonical.y, metadata.pixelRatio);
|
|
|
|
const auto& placedSymbols = ctx.getFrontend().getRenderer()->getPlacedSymbolsData();
|
|
|
|
auto findCutOffs =
|
|
[&](const PlacedSymbolData& placedSymbol, mapbox::geometry::box<float> box, bool placed, bool isIcon) {
|
|
box.min.x += xOffset - placedSymbol.viewportPadding;
|
|
box.max.x += xOffset - placedSymbol.viewportPadding;
|
|
box.min.y += yOffset - placedSymbol.viewportPadding;
|
|
box.max.y += yOffset - placedSymbol.viewportPadding;
|
|
|
|
auto& symbols = symbolIndex[placedSymbol.key];
|
|
auto it = std::find_if(symbols.begin(), symbols.end(), [isIcon, box](const auto& a) {
|
|
return a.isIcon == isIcon && a.location.min == box.min && a.location.max == box.max;
|
|
});
|
|
|
|
// TODO: remove usage of std::codecvt_utf8
|
|
// https://github.com/maplibre/maplibre-native/issues/1269
|
|
#if defined(__clang__)
|
|
#pragma clang diagnostic push
|
|
#pragma clang diagnostic ignored "-Wdeprecated-declarations"
|
|
#endif
|
|
#if defined(__GNUC__)
|
|
#pragma GCC diagnostic push
|
|
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
|
|
#endif
|
|
static std::wstring_convert<std::codecvt_utf8<char16_t>, char16_t> cv;
|
|
#if defined(__GNUC__)
|
|
#pragma GCC diagnostic pop
|
|
#endif
|
|
#if defined(__clang__)
|
|
#pragma clang diagnostic pop
|
|
#endif
|
|
if (it == symbols.end()) {
|
|
symbols.push_back({tileId, box, placed, isIcon});
|
|
return;
|
|
}
|
|
if (tileId == it->tileId && isIcon == it->isIcon) {
|
|
std::stringstream ss;
|
|
ss << std::endl
|
|
<< ++metadata.duplicationsCount << ". \"" << cv.to_bytes(placedSymbol.key) << "\" "
|
|
<< (isIcon ? "icon" : "text") << " ((" << box.min.x << ", " << box.min.y << "), ("
|
|
<< box.max.x << ", " << box.max.y << ")) from layer: " << placedSymbol.layer << " at "
|
|
<< util::toString(tileId);
|
|
duplicationsReport += ss.str();
|
|
return;
|
|
}
|
|
if (it->placed != placed && isIcon == it->isIcon) {
|
|
std::stringstream ss;
|
|
ss << std::endl
|
|
<< ++metadata.labelCutOffFound << ". \"" << cv.to_bytes(placedSymbol.key) << "\" "
|
|
<< (isIcon ? "icon" : "text") << " ((" << box.min.x << ", " << box.min.y << "), ("
|
|
<< box.max.x << ", " << box.max.y << ")) from layer: " << placedSymbol.layer
|
|
<< (placed ? " is placed at " : " is not placed at ") << util::toString(tileId) << " and"
|
|
<< (it->placed ? " placed at " : " not placed at ") << util::toString(it->tileId);
|
|
cutOffLabelsReport += ss.str();
|
|
}
|
|
};
|
|
|
|
for (const auto& placedSymbol : placedSymbols) {
|
|
if (placedSymbol.intersectsTileBorder) {
|
|
if (placedSymbol.textCollisionBox) {
|
|
findCutOffs(
|
|
placedSymbol, *placedSymbol.textCollisionBox, placedSymbol.textPlaced, false /*isIcon*/
|
|
);
|
|
}
|
|
if (placedSymbol.iconCollisionBox) {
|
|
findCutOffs(
|
|
placedSymbol, *placedSymbol.iconCollisionBox, placedSymbol.iconPlaced, true /*isIcon*/
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
PremultipliedImage::copy(
|
|
resultForTile.image, result.image, {0, 0}, {xOffset, yOffset}, resultForTile.image.size);
|
|
result.stats += resultForTile.stats;
|
|
}
|
|
} else {
|
|
map.jumpTo(camera);
|
|
result = runTest(metadata, ctx);
|
|
}
|
|
|
|
if (!metadata.ignoredTest) {
|
|
ctx.activeGfxProbe = GfxProbe(result.stats, ctx.activeGfxProbe);
|
|
for (const auto& operation : getAfterOperations(manifest)) {
|
|
if (!operation(ctx)) return;
|
|
}
|
|
}
|
|
|
|
if (metadata.renderTest) {
|
|
if (!metadata.ignoreProbing) {
|
|
checkProbingResults(metadata);
|
|
}
|
|
appendLabelCutOffResults(metadata, cutOffLabelsReport, duplicationsReport);
|
|
checkRenderTestResults(std::move(result.image), metadata);
|
|
} else {
|
|
std::vector<mbgl::Feature> features;
|
|
assert(metadata.document["metadata"]["test"]["queryGeometry"].IsArray());
|
|
if (metadata.document["metadata"]["test"]["queryGeometry"][0].IsNumber() &&
|
|
metadata.document["metadata"]["test"]["queryGeometry"][1].IsNumber()) {
|
|
features = frontend.getRenderer()->queryRenderedFeatures(metadata.queryGeometry, metadata.queryOptions);
|
|
} else {
|
|
features = frontend.getRenderer()->queryRenderedFeatures(metadata.queryGeometryBox, metadata.queryOptions);
|
|
}
|
|
checkQueryTestResults(std::move(result.image), std::move(features), metadata);
|
|
}
|
|
}
|
|
|
|
void TestRunner::appendLabelCutOffResults(TestMetadata& resultMetadata,
|
|
const std::string& cutOffLabelsReport,
|
|
const std::string& duplicationsReport) {
|
|
if (resultMetadata.labelCutOffFound) { // Append label cut-off statistics.
|
|
resultMetadata.errorMessage += "\n Label cut-offs:";
|
|
resultMetadata.errorMessage += cutOffLabelsReport;
|
|
if (resultMetadata.duplicationsCount) {
|
|
resultMetadata.errorMessage += "\n\n Label duplications:";
|
|
resultMetadata.errorMessage += duplicationsReport;
|
|
}
|
|
}
|
|
}
|
|
|
|
mbgl::HeadlessFrontend::RenderResult TestRunner::runTest(TestMetadata& metadata, TestContext& ctx) {
|
|
HeadlessFrontend::RenderResult result{};
|
|
for (const auto& operation : parseTestOperations(metadata)) {
|
|
if (!operation(ctx)) return result;
|
|
}
|
|
|
|
try {
|
|
if (metadata.outputsImage) result = ctx.getFrontend().render(ctx.getMap());
|
|
} catch (const std::exception& e) {
|
|
ctx.getMetadata().errorMessage = std::string("Renering raised an exception: ") + e.what();
|
|
}
|
|
return result;
|
|
}
|
|
|
|
void TestRunner::reset() {
|
|
maps.clear();
|
|
}
|