[test-runner] Introduce mapbox-gl-native test expectations
The `mapbox-gl-native`-specific render tests expected results will be stored at the `render-test/expected/` subfolder. Test runner will automatically put new expected results there if invoked with `UPDATE=true`.
This commit is contained in:
@@ -7,6 +7,7 @@
|
||||
#include "runner.hpp"
|
||||
|
||||
#include <random>
|
||||
#include <regex>
|
||||
|
||||
#define ANSI_COLOR_RED "\x1b[31m"
|
||||
#define ANSI_COLOR_GREEN "\x1b[32m"
|
||||
@@ -18,6 +19,25 @@
|
||||
#define ANSI_COLOR_LIGHT_GRAY "\x1b[90m"
|
||||
#define ANSI_COLOR_RESET "\x1b[0m"
|
||||
|
||||
namespace {
|
||||
|
||||
TestPaths makeTestPaths(mbgl::filesystem::path stylePath) {
|
||||
auto defaultExpectations = stylePath;
|
||||
defaultExpectations.remove_filename();
|
||||
const static std::regex regex{ TestRunner::getBasePath() };
|
||||
auto platformExpectations = std::regex_replace(defaultExpectations.string(), regex, TestRunner::getPlatformExpectationsPath());
|
||||
assert(!defaultExpectations.empty());
|
||||
assert(!platformExpectations.empty());
|
||||
|
||||
return {
|
||||
std::move(stylePath),
|
||||
std::move(defaultExpectations),
|
||||
std::move(platformExpectations)
|
||||
};
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
bool recycleMap;
|
||||
bool shuffle;
|
||||
@@ -31,11 +51,12 @@ int main(int argc, char** argv) {
|
||||
const auto ignores = parseIgnores();
|
||||
|
||||
// Recursively traverse through the test paths and collect test directories containing "style.json".
|
||||
std::vector<mbgl::filesystem::path> testPaths;
|
||||
std::vector<TestPaths> testPaths;
|
||||
testPaths.reserve(ids.size());
|
||||
for (const auto& id : ids) {
|
||||
for (auto& testPath : mbgl::filesystem::recursive_directory_iterator(mbgl::filesystem::path(id))) {
|
||||
if (testPath.path().filename() == "style.json") {
|
||||
testPaths.push_back(testPath);
|
||||
testPaths.emplace_back(makeTestPaths(testPath));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -67,7 +88,7 @@ int main(int argc, char** argv) {
|
||||
std::string& status = metadata.status;
|
||||
std::string& color = metadata.color;
|
||||
|
||||
id = testPath.remove_filename().string();
|
||||
id = testPath.defaultExpectations.string();
|
||||
id = id.substr(rootLength + 1, id.length() - rootLength - 2);
|
||||
|
||||
bool shouldIgnore = false;
|
||||
|
||||
@@ -17,10 +17,16 @@ struct TestStatistics {
|
||||
uint32_t passedTests = 0;
|
||||
};
|
||||
|
||||
struct TestPaths {
|
||||
mbgl::filesystem::path stylePath;
|
||||
mbgl::filesystem::path defaultExpectations;
|
||||
mbgl::filesystem::path platformExpectations;
|
||||
};
|
||||
|
||||
struct TestMetadata {
|
||||
TestMetadata() = default;
|
||||
|
||||
mbgl::filesystem::path path;
|
||||
TestPaths paths;
|
||||
mbgl::JSDocument document;
|
||||
|
||||
mbgl::Size size{ 512u, 512u };
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
|
||||
#include "parser.hpp"
|
||||
#include "metadata.hpp"
|
||||
#include "runner.hpp"
|
||||
|
||||
#include <sstream>
|
||||
#include <regex>
|
||||
@@ -243,17 +244,18 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
|
||||
|
||||
std::vector<std::string> ids;
|
||||
for (const auto& id : args::get(testNameValues)) {
|
||||
ids.emplace_back(testDefaultPath + "/" + id);
|
||||
ids.emplace_back(TestRunner::getBasePath() + "/" + id);
|
||||
}
|
||||
|
||||
if (ids.empty()) {
|
||||
ids.emplace_back(testDefaultPath);
|
||||
ids.emplace_back(TestRunner::getBasePath());
|
||||
}
|
||||
|
||||
return ArgumentsTuple {
|
||||
recycleMapFlag ? args::get(recycleMapFlag) : false,
|
||||
shuffleFlag ? args::get(shuffleFlag) : false, seedValue ? args::get(seedValue) : 1u,
|
||||
testPathValue ? args::get(testPathValue) : testDefaultPath, ids
|
||||
testPathValue ? args::get(testPathValue) : testDefaultPath,
|
||||
std::move(ids)
|
||||
};
|
||||
}
|
||||
|
||||
@@ -288,13 +290,13 @@ std::vector<std::pair<std::string, std::string>> parseIgnores() {
|
||||
return ignores;
|
||||
}
|
||||
|
||||
TestMetadata parseTestMetadata(const mbgl::filesystem::path& path) {
|
||||
TestMetadata parseTestMetadata(const TestPaths& paths) {
|
||||
TestMetadata metadata;
|
||||
metadata.path = path;
|
||||
metadata.paths = paths;
|
||||
|
||||
auto maybeJson = readJson(path.string());
|
||||
auto maybeJson = readJson(paths.stylePath.string());
|
||||
if (!maybeJson.is<mbgl::JSDocument>()) { // NOLINT
|
||||
metadata.errorMessage = std::string("Unable to parse: ") + path.string();
|
||||
metadata.errorMessage = std::string("Unable to parse: ") + metadata.paths.stylePath.string();
|
||||
return metadata;
|
||||
}
|
||||
|
||||
@@ -303,14 +305,14 @@ TestMetadata parseTestMetadata(const mbgl::filesystem::path& path) {
|
||||
|
||||
if (!metadata.document.HasMember("metadata")) {
|
||||
mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata': %s",
|
||||
path.c_str());
|
||||
paths.stylePath.c_str());
|
||||
return metadata;
|
||||
}
|
||||
|
||||
const mbgl::JSValue& metadataValue = metadata.document["metadata"];
|
||||
if (!metadataValue.HasMember("test")) {
|
||||
mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata.test': %s",
|
||||
path.c_str());
|
||||
paths.stylePath.c_str());
|
||||
return metadata;
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
|
||||
struct TestMetadata;
|
||||
struct TestStatistics;
|
||||
struct TestPaths;
|
||||
|
||||
using ErrorMessage = std::string;
|
||||
using JSONReply = mbgl::variant<mbgl::JSDocument, ErrorMessage>;
|
||||
@@ -24,7 +25,8 @@ std::vector<std::string> readExpectedEntries(const mbgl::filesystem::path& base)
|
||||
|
||||
ArgumentsTuple parseArguments(int argc, char** argv);
|
||||
std::vector<std::pair<std::string, std::string>> parseIgnores();
|
||||
TestMetadata parseTestMetadata(const mbgl::filesystem::path& path);
|
||||
|
||||
TestMetadata parseTestMetadata(const TestPaths& paths);
|
||||
|
||||
std::string createResultPage(const TestStatistics&, const std::vector<TestMetadata>&, bool shuffle, uint32_t seed);
|
||||
|
||||
|
||||
@@ -26,8 +26,24 @@
|
||||
#include <cassert>
|
||||
#include <regex>
|
||||
|
||||
|
||||
// static
|
||||
const std::string& TestRunner::getBasePath() {
|
||||
const static std::string result =
|
||||
std::string(TEST_RUNNER_ROOT_PATH).append("/mapbox-gl-js/test/integration/render-tests");
|
||||
return result;
|
||||
}
|
||||
|
||||
// static
|
||||
const std::string& TestRunner::getPlatformExpectationsPath() {
|
||||
const static std::string result =
|
||||
std::string(TEST_RUNNER_ROOT_PATH).append("/render-test/expected");
|
||||
return result;
|
||||
}
|
||||
|
||||
bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& metadata) {
|
||||
const std::string base = metadata.path.remove_filename().string();
|
||||
const std::string& base = metadata.paths.defaultExpectations.string();
|
||||
|
||||
metadata.actual = mbgl::encodePNG(actual);
|
||||
|
||||
if (actual.size.isEmpty()) {
|
||||
@@ -37,7 +53,8 @@ bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& met
|
||||
|
||||
#if !TEST_READ_ONLY
|
||||
if (getenv("UPDATE")) {
|
||||
mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actual));
|
||||
mbgl::filesystem::create_directories(metadata.paths.platformExpectations);
|
||||
mbgl::util::write_file(metadata.paths.platformExpectations.string() + "/expected.png", mbgl::encodePNG(actual));
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -48,8 +65,10 @@ bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& met
|
||||
mbgl::PremultipliedImage diff { actual.size };
|
||||
|
||||
double pixels = 0.0;
|
||||
const auto& expectedPath = mbgl::filesystem::exists(metadata.paths.platformExpectations) ?
|
||||
metadata.paths.platformExpectations : metadata.paths.defaultExpectations;
|
||||
|
||||
for (const auto& entry: readExpectedEntries(base)) {
|
||||
for (const auto& entry: readExpectedEntries(expectedPath)) {
|
||||
mbgl::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry);
|
||||
if (!maybeExpectedImage) {
|
||||
metadata.errorMessage = "Failed to load expected image " + entry;
|
||||
|
||||
@@ -14,6 +14,11 @@ public:
|
||||
bool run(TestMetadata&);
|
||||
void reset();
|
||||
|
||||
/// Returns path of the render tests root directory.
|
||||
static const std::string& getBasePath();
|
||||
/// Returns path of mapbox-gl-native expectations directory.
|
||||
static const std::string& getPlatformExpectationsPath();
|
||||
|
||||
private:
|
||||
bool runOperations(const std::string& key, TestMetadata&);
|
||||
bool checkImage(mbgl::PremultipliedImage&& image, TestMetadata&);
|
||||
|
||||
Reference in New Issue
Block a user