[render-test] Add manifest parser for render-test-runner (#15861)

* [render-test] Add manifest parser for render-test-runner

* [render-test] Refactory manifest parser

* [render-test] Parse full manifest file through CLI

* [render-test] Add linux probe manifest
This commit is contained in:
zmiao
2019-10-29 20:59:29 +02:00
committed by GitHub
parent 84f2ff1084
commit c8576e089f
19 changed files with 634 additions and 460 deletions

View File

@@ -2,14 +2,16 @@
#include <mbgl/render_test.hpp>
#include <mbgl/util/io.hpp>
#include <mbgl/util/logging.hpp>
#include <mbgl/util/run_loop.hpp>
#include <args.hxx>
#include "manifest_parser.hpp"
#include "metadata.hpp"
#include "parser.hpp"
#include "runner.hpp"
#include <random>
#define ANSI_COLOR_RED "\x1b[31m"
#define ANSI_COLOR_GREEN "\x1b[32m"
#define ANSI_COLOR_YELLOW "\x1b[33m"
@@ -37,38 +39,97 @@ void operator delete(void* ptr, size_t) noexcept {
}
#endif
namespace {
using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::vector<std::string>, std::string>;
ArgumentsTuple parseArguments(int argc, char** argv) {
args::ArgumentParser argumentParser("Mapbox GL Test Runner");
args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", {'h', "help"});
args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object", {'r', "recycle-map"});
args::Flag shuffleFlag(argumentParser, "shuffle", "Toggle shuffling the tests order", {'s', "shuffle"});
args::ValueFlag<uint32_t> seedValue(argumentParser, "seed", "Shuffle seed (default: random)", {"seed"});
args::ValueFlag<std::string> testPathValue(
argumentParser, "manifestPath", "Test manifest file path", {'p', "manifestPath"});
args::ValueFlag<std::string> testFilterValue(argumentParser, "filter", "Test filter regex", {'f', "filter"});
args::PositionalList<std::string> testNameValues(argumentParser, "URL", "Test name(s)");
try {
argumentParser.ParseCLI(argc, argv);
} catch (const args::Help&) {
std::ostringstream stream;
stream << argumentParser;
mbgl::Log::Info(mbgl::Event::General, stream.str());
exit(0);
} catch (const args::ParseError& e) {
std::ostringstream stream;
stream << argumentParser;
mbgl::Log::Info(mbgl::Event::General, stream.str());
mbgl::Log::Error(mbgl::Event::General, e.what());
exit(1);
} catch (const args::ValidationError& e) {
std::ostringstream stream;
stream << argumentParser;
mbgl::Log::Info(mbgl::Event::General, stream.str());
mbgl::Log::Error(mbgl::Event::General, e.what());
exit(2);
} catch (const std::regex_error& e) {
mbgl::Log::Error(mbgl::Event::General, "Invalid filter regular expression: %s", e.what());
exit(3);
}
mbgl::filesystem::path manifestPath{testPathValue ? args::get(testPathValue) : std::string{TEST_RUNNER_ROOT_PATH}};
if (!mbgl::filesystem::exists(manifestPath) || !manifestPath.has_filename()) {
mbgl::Log::Error(mbgl::Event::General,
"Provided test manifest file path '%s' does not exist",
manifestPath.string().c_str());
exit(4);
}
auto testNames = testNameValues ? args::get(testNameValues) : std::vector<std::string>{};
auto testFilter = testFilterValue ? args::get(testFilterValue) : std::string{};
const auto shuffle = shuffleFlag ? args::get(shuffleFlag) : false;
const auto seed = seedValue ? args::get(seedValue) : 1u;
return ArgumentsTuple{recycleMapFlag ? args::get(recycleMapFlag) : false,
shuffle,
seed,
manifestPath.string(),
std::move(testNames),
std::move(testFilter)};
}
} // namespace
namespace mbgl {
int runRenderTests(int argc, char** argv) {
bool recycleMap;
bool shuffle;
uint32_t seed;
std::string testRootPath;
std::string ignoresPath;
std::vector<TestPaths> testPaths;
std::tie(recycleMap, shuffle, seed, testRootPath, ignoresPath, testPaths) = parseArguments(argc, argv);
const auto ignores = parseIgnores(testRootPath, ignoresPath);
std::string manifestPath;
std::vector<std::string> testNames;
std::string testFilter;
std::tie(recycleMap, shuffle, seed, manifestPath, testNames, testFilter) = parseArguments(argc, argv);
auto manifestData = ManifestParser::parseManifest(manifestPath, testNames, testFilter);
if (!manifestData) {
exit(5);
}
mbgl::util::RunLoop runLoop;
TestRunner runner(std::move(*manifestData));
if (shuffle) {
printf(ANSI_COLOR_YELLOW "Shuffle seed: %d" ANSI_COLOR_RESET "\n", seed);
std::seed_seq sequence { seed };
std::mt19937 shuffler(sequence);
std::shuffle(testPaths.begin(), testPaths.end(), shuffler);
runner.doShuffle(seed);
}
mbgl::util::RunLoop runLoop;
TestRunner runner(testRootPath);
const auto& manifest = runner.getManifest();
const auto& ignores = manifest.getIgnores();
const auto& testPaths = manifest.getTestPaths();
std::vector<TestMetadata> metadatas;
metadatas.reserve(testPaths.size());
TestStatistics stats;
for (auto& testPath : testPaths) {
TestMetadata metadata = parseTestMetadata(testPath, testRootPath);
TestMetadata metadata = parseTestMetadata(testPath, manifest);
if (!recycleMap) {
runner.reset();
@@ -78,7 +139,7 @@ int runRenderTests(int argc, char** argv) {
std::string& status = metadata.status;
std::string& color = metadata.color;
const std::string::size_type rootLength = getTestPath(testRootPath).length();
const std::string::size_type rootLength = manifest.getTestRootPath().length();
id = testPath.defaultExpectations();
id = id.substr(rootLength + 1, id.length() - rootLength - 2);
@@ -138,13 +199,14 @@ int runRenderTests(int argc, char** argv) {
metadatas.push_back(std::move(metadata));
}
const auto& testRootPath = manifest.getManifestPath();
const auto resultPath =
testRootPath + "/" + (testNames.empty() ? "render-tests" : testNames.front()) + "_index.html";
std::string resultsHTML = createResultPage(stats, metadatas, shuffle, seed);
mbgl::util::write_file(testRootPath + "/index.html", resultsHTML);
mbgl::util::write_file(resultPath, resultsHTML);
const uint32_t count = stats.erroredTests + stats.failedTests +
stats.ignoreFailedTests + stats.ignorePassedTests +
stats.passedTests;
const uint32_t count =
stats.erroredTests + stats.failedTests + stats.ignoreFailedTests + stats.ignorePassedTests + stats.passedTests;
if (stats.passedTests) {
printf(ANSI_COLOR_GREEN "%u passed (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.passedTests, 100.0 * stats.passedTests / count);
@@ -162,7 +224,7 @@ int runRenderTests(int argc, char** argv) {
printf(ANSI_COLOR_RED "%u errored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.erroredTests, 100.0 * stats.erroredTests / count);
}
printf("Results at: %s%s\n", testRootPath.c_str(), "/index.html");
printf("Results at: %s\n", resultPath.c_str());
return stats.failedTests + stats.erroredTests == 0 ? 0 : 1;
}