[test-runner] Introduce mapbox-gl-native test expectations

The `mapbox-gl-native`-specific render tests expected results will
be stored at the `render-test/expected/` subfolder.

Test runner will automatically put new expected results there if
invoked with `UPDATE=true`.
This commit is contained in:
Mikhail Pozdnyakov
2019-08-14 17:07:00 +03:00
parent 1165f62df6
commit a1904e802c
6 changed files with 72 additions and 17 deletions

View File

@@ -7,6 +7,7 @@
#include "runner.hpp"
#include <random>
#include <regex>
#define ANSI_COLOR_RED "\x1b[31m"
#define ANSI_COLOR_GREEN "\x1b[32m"
@@ -18,6 +19,25 @@
#define ANSI_COLOR_LIGHT_GRAY "\x1b[90m"
#define ANSI_COLOR_RESET "\x1b[0m"
namespace {
TestPaths makeTestPaths(mbgl::filesystem::path stylePath) {
auto defaultExpectations = stylePath;
defaultExpectations.remove_filename();
const static std::regex regex{ TestRunner::getBasePath() };
auto platformExpectations = std::regex_replace(defaultExpectations.string(), regex, TestRunner::getPlatformExpectationsPath());
assert(!defaultExpectations.empty());
assert(!platformExpectations.empty());
return {
std::move(stylePath),
std::move(defaultExpectations),
std::move(platformExpectations)
};
}
} // namespace
int main(int argc, char** argv) {
bool recycleMap;
bool shuffle;
@@ -31,11 +51,12 @@ int main(int argc, char** argv) {
const auto ignores = parseIgnores();
// Recursively traverse through the test paths and collect test directories containing "style.json".
std::vector<mbgl::filesystem::path> testPaths;
std::vector<TestPaths> testPaths;
testPaths.reserve(ids.size());
for (const auto& id : ids) {
for (auto& testPath : mbgl::filesystem::recursive_directory_iterator(mbgl::filesystem::path(id))) {
if (testPath.path().filename() == "style.json") {
testPaths.push_back(testPath);
testPaths.emplace_back(makeTestPaths(testPath));
}
}
}
@@ -67,7 +88,7 @@ int main(int argc, char** argv) {
std::string& status = metadata.status;
std::string& color = metadata.color;
id = testPath.remove_filename().string();
id = testPath.defaultExpectations.string();
id = id.substr(rootLength + 1, id.length() - rootLength - 2);
bool shouldIgnore = false;

View File

@@ -17,10 +17,16 @@ struct TestStatistics {
uint32_t passedTests = 0;
};
struct TestPaths {
mbgl::filesystem::path stylePath;
mbgl::filesystem::path defaultExpectations;
mbgl::filesystem::path platformExpectations;
};
struct TestMetadata {
TestMetadata() = default;
mbgl::filesystem::path path;
TestPaths paths;
mbgl::JSDocument document;
mbgl::Size size{ 512u, 512u };

View File

@@ -15,6 +15,7 @@
#include "parser.hpp"
#include "metadata.hpp"
#include "runner.hpp"
#include <sstream>
#include <regex>
@@ -243,17 +244,18 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
std::vector<std::string> ids;
for (const auto& id : args::get(testNameValues)) {
ids.emplace_back(testDefaultPath + "/" + id);
ids.emplace_back(TestRunner::getBasePath() + "/" + id);
}
if (ids.empty()) {
ids.emplace_back(testDefaultPath);
ids.emplace_back(TestRunner::getBasePath());
}
return ArgumentsTuple {
recycleMapFlag ? args::get(recycleMapFlag) : false,
shuffleFlag ? args::get(shuffleFlag) : false, seedValue ? args::get(seedValue) : 1u,
testPathValue ? args::get(testPathValue) : testDefaultPath, ids
testPathValue ? args::get(testPathValue) : testDefaultPath,
std::move(ids)
};
}
@@ -288,13 +290,13 @@ std::vector<std::pair<std::string, std::string>> parseIgnores() {
return ignores;
}
TestMetadata parseTestMetadata(const mbgl::filesystem::path& path) {
TestMetadata parseTestMetadata(const TestPaths& paths) {
TestMetadata metadata;
metadata.path = path;
metadata.paths = paths;
auto maybeJson = readJson(path.string());
auto maybeJson = readJson(paths.stylePath.string());
if (!maybeJson.is<mbgl::JSDocument>()) { // NOLINT
metadata.errorMessage = std::string("Unable to parse: ") + path.string();
metadata.errorMessage = std::string("Unable to parse: ") + metadata.paths.stylePath.string();
return metadata;
}
@@ -303,14 +305,14 @@ TestMetadata parseTestMetadata(const mbgl::filesystem::path& path) {
if (!metadata.document.HasMember("metadata")) {
mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata': %s",
path.c_str());
paths.stylePath.c_str());
return metadata;
}
const mbgl::JSValue& metadataValue = metadata.document["metadata"];
if (!metadataValue.HasMember("test")) {
mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata.test': %s",
path.c_str());
paths.stylePath.c_str());
return metadata;
}

View File

@@ -11,6 +11,7 @@
struct TestMetadata;
struct TestStatistics;
struct TestPaths;
using ErrorMessage = std::string;
using JSONReply = mbgl::variant<mbgl::JSDocument, ErrorMessage>;
@@ -24,7 +25,8 @@ std::vector<std::string> readExpectedEntries(const mbgl::filesystem::path& base)
ArgumentsTuple parseArguments(int argc, char** argv);
std::vector<std::pair<std::string, std::string>> parseIgnores();
TestMetadata parseTestMetadata(const mbgl::filesystem::path& path);
TestMetadata parseTestMetadata(const TestPaths& paths);
std::string createResultPage(const TestStatistics&, const std::vector<TestMetadata>&, bool shuffle, uint32_t seed);

View File

@@ -26,8 +26,24 @@
#include <cassert>
#include <regex>
// static
const std::string& TestRunner::getBasePath() {
const static std::string result =
std::string(TEST_RUNNER_ROOT_PATH).append("/mapbox-gl-js/test/integration/render-tests");
return result;
}
// static
const std::string& TestRunner::getPlatformExpectationsPath() {
const static std::string result =
std::string(TEST_RUNNER_ROOT_PATH).append("/render-test/expected");
return result;
}
bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& metadata) {
const std::string base = metadata.path.remove_filename().string();
const std::string& base = metadata.paths.defaultExpectations.string();
metadata.actual = mbgl::encodePNG(actual);
if (actual.size.isEmpty()) {
@@ -37,7 +53,8 @@ bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& met
#if !TEST_READ_ONLY
if (getenv("UPDATE")) {
mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actual));
mbgl::filesystem::create_directories(metadata.paths.platformExpectations);
mbgl::util::write_file(metadata.paths.platformExpectations.string() + "/expected.png", mbgl::encodePNG(actual));
return true;
}
@@ -48,8 +65,10 @@ bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& met
mbgl::PremultipliedImage diff { actual.size };
double pixels = 0.0;
const auto& expectedPath = mbgl::filesystem::exists(metadata.paths.platformExpectations) ?
metadata.paths.platformExpectations : metadata.paths.defaultExpectations;
for (const auto& entry: readExpectedEntries(base)) {
for (const auto& entry: readExpectedEntries(expectedPath)) {
mbgl::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry);
if (!maybeExpectedImage) {
metadata.errorMessage = "Failed to load expected image " + entry;

View File

@@ -14,6 +14,11 @@ public:
bool run(TestMetadata&);
void reset();
/// Returns path of the render tests root directory.
static const std::string& getBasePath();
/// Returns path of mapbox-gl-native expectations directory.
static const std::string& getPlatformExpectationsPath();
private:
bool runOperations(const std::string& key, TestMetadata&);
bool checkImage(mbgl::PremultipliedImage&& image, TestMetadata&);