[general] moving ios and android project under ./platform folder
This commit is contained in:
Petr Pokorny
273
render-test/render_test.cpp
Normal file
273
render-test/render_test.cpp
Normal file
@@ -0,0 +1,273 @@
|
||||
#include "allocation_index.hpp"
|
||||
|
||||
#include <mbgl/render_test.hpp>
|
||||
#include <mbgl/storage/network_status.hpp>
|
||||
#include <mbgl/util/io.hpp>
|
||||
#include <mbgl/util/logging.hpp>
|
||||
#include <mbgl/util/run_loop.hpp>
|
||||
|
||||
#include <args.hxx>
|
||||
|
||||
#include "file_source.hpp"
|
||||
#include "manifest_parser.hpp"
|
||||
#include "metadata.hpp"
|
||||
#include "parser.hpp"
|
||||
#include "runner.hpp"
|
||||
|
||||
#define ANSI_COLOR_RED "\x1b[31m"
|
||||
#define ANSI_COLOR_GREEN "\x1b[32m"
|
||||
#define ANSI_COLOR_YELLOW "\x1b[33m"
|
||||
#define ANSI_COLOR_BLUE "\x1b[34m"
|
||||
#define ANSI_COLOR_MAGENTA "\x1b[35m"
|
||||
#define ANSI_COLOR_CYAN "\x1b[36m"
|
||||
#define ANSI_COLOR_GRAY "\x1b[37m"
|
||||
#define ANSI_COLOR_LIGHT_GRAY "\x1b[90m"
|
||||
#define ANSI_COLOR_RESET "\x1b[0m"
|
||||
|
||||
#if !defined(SANITIZE)
|
||||
void* operator new(std::size_t sz) {
|
||||
void* ptr = AllocationIndex::allocate(sz);
|
||||
if (!ptr) throw std::bad_alloc{};
|
||||
|
||||
return ptr;
|
||||
}
|
||||
|
||||
void operator delete(void* ptr) noexcept {
|
||||
AllocationIndex::deallocate(ptr);
|
||||
}
|
||||
|
||||
void operator delete(void* ptr, size_t) noexcept {
|
||||
AllocationIndex::deallocate(ptr);
|
||||
}
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
|
||||
using ArgumentsTuple = std::tuple<bool, bool, bool, uint32_t, std::string, TestRunner::UpdateResults, std::string>;
|
||||
ArgumentsTuple parseArguments(int argc, char** argv) {
|
||||
const static std::unordered_map<std::string, TestRunner::UpdateResults> updateResultsFlags = {
|
||||
{"default", TestRunner::UpdateResults::DEFAULT},
|
||||
{"platform", TestRunner::UpdateResults::PLATFORM},
|
||||
{"metrics", TestRunner::UpdateResults::METRICS},
|
||||
{"rebaseline", TestRunner::UpdateResults::REBASELINE}};
|
||||
|
||||
args::ArgumentParser argumentParser("Mapbox GL Test Runner");
|
||||
|
||||
args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", {'h', "help"});
|
||||
|
||||
args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object", {'r', "recycle-map"});
|
||||
args::Flag shuffleFlag(argumentParser, "shuffle", "Toggle shuffling the tests order", {'s', "shuffle"});
|
||||
args::Flag onlineFlag(
|
||||
argumentParser, "online", "Toggle online mode (by default tests will run offline)", {'o', "online"});
|
||||
args::ValueFlag<uint32_t> seedValue(argumentParser, "seed", "Shuffle seed (default: random)", {"seed"});
|
||||
args::ValueFlag<std::string> testPathValue(
|
||||
argumentParser, "manifestPath", "Test manifest file path", {'p', "manifestPath"}, args::Options::Required);
|
||||
args::ValueFlag<std::string> testFilterValue(argumentParser, "filter", "Test filter regex", {'f', "filter"});
|
||||
args::MapFlag<std::string, TestRunner::UpdateResults> testUpdateResultsValue(
|
||||
argumentParser,
|
||||
"update",
|
||||
"Test results update mode. Supported values are: \n\"default\" Updates generic render test expectations. \
|
||||
\n\"platform\" Updates platform specific render test expectations. \
|
||||
\n\"metrics\" Updates expected metrics for configuration defined by a manifest.\
|
||||
\n\"rebaseline\" Updates or creates expected metrics for configuration defined by a manifest.",
|
||||
{'u', "update"},
|
||||
updateResultsFlags);
|
||||
|
||||
try {
|
||||
argumentParser.ParseCLI(argc, argv);
|
||||
} catch (const args::Help&) {
|
||||
std::ostringstream stream;
|
||||
stream << argumentParser;
|
||||
mbgl::Log::Info(mbgl::Event::General, stream.str());
|
||||
exit(0);
|
||||
} catch (const args::ParseError& e) {
|
||||
std::ostringstream stream;
|
||||
stream << argumentParser;
|
||||
mbgl::Log::Info(mbgl::Event::General, stream.str());
|
||||
mbgl::Log::Error(mbgl::Event::General, e.what());
|
||||
exit(1);
|
||||
} catch (const args::ValidationError& e) {
|
||||
std::ostringstream stream;
|
||||
stream << argumentParser;
|
||||
mbgl::Log::Info(mbgl::Event::General, stream.str());
|
||||
mbgl::Log::Error(mbgl::Event::General, e.what());
|
||||
exit(2);
|
||||
}
|
||||
|
||||
mbgl::filesystem::path manifestPath = args::get(testPathValue);
|
||||
if (!mbgl::filesystem::exists(manifestPath) || !manifestPath.has_filename()) {
|
||||
mbgl::Log::Error(mbgl::Event::General,
|
||||
"Provided test manifest file path '%s' does not exist",
|
||||
manifestPath.string().c_str());
|
||||
exit(3);
|
||||
}
|
||||
|
||||
auto testFilter = testFilterValue ? args::get(testFilterValue) : std::string{};
|
||||
const auto shuffle = shuffleFlag ? args::get(shuffleFlag) : false;
|
||||
const auto online = onlineFlag ? args::get(onlineFlag) : false;
|
||||
const auto seed = seedValue ? args::get(seedValue) : 1u;
|
||||
TestRunner::UpdateResults updateResults =
|
||||
testUpdateResultsValue ? args::get(testUpdateResultsValue) : TestRunner::UpdateResults::NO;
|
||||
return ArgumentsTuple{recycleMapFlag ? args::get(recycleMapFlag) : false,
|
||||
shuffle,
|
||||
online,
|
||||
seed,
|
||||
manifestPath.string(),
|
||||
updateResults,
|
||||
std::move(testFilter)};
|
||||
}
|
||||
} // namespace
|
||||
namespace mbgl {
|
||||
|
||||
int runRenderTests(int argc, char** argv, std::function<void()> testStatus) {
|
||||
int returnCode = 0;
|
||||
bool recycleMap;
|
||||
bool shuffle;
|
||||
bool online;
|
||||
uint32_t seed;
|
||||
std::string manifestPath;
|
||||
std::string testFilter;
|
||||
|
||||
Log::useLogThread(false);
|
||||
TestRunner::UpdateResults updateResults;
|
||||
|
||||
std::tie(recycleMap, shuffle, online, seed, manifestPath, updateResults, testFilter) = parseArguments(argc, argv);
|
||||
|
||||
ProxyFileSource::setOffline(!online);
|
||||
|
||||
auto manifestData = ManifestParser::parseManifest(manifestPath, testFilter);
|
||||
if (!manifestData) {
|
||||
exit(5);
|
||||
}
|
||||
mbgl::util::RunLoop runLoop;
|
||||
TestRunner runner(std::move(*manifestData), updateResults);
|
||||
if (shuffle) {
|
||||
printf(ANSI_COLOR_YELLOW "Shuffle seed: %d" ANSI_COLOR_RESET "\n", seed);
|
||||
runner.doShuffle(seed);
|
||||
}
|
||||
|
||||
NetworkStatus::Set(online ? NetworkStatus::Status::Online : NetworkStatus::Status::Offline);
|
||||
|
||||
const auto& manifest = runner.getManifest();
|
||||
const auto& ignores = manifest.getIgnores();
|
||||
const auto& testPaths = manifest.getTestPaths();
|
||||
std::vector<TestMetadata> metadatas;
|
||||
metadatas.reserve(testPaths.size());
|
||||
|
||||
TestStatistics stats;
|
||||
|
||||
for (auto& testPath : testPaths) {
|
||||
TestMetadata metadata = parseTestMetadata(testPath);
|
||||
|
||||
if (!recycleMap) {
|
||||
runner.reset();
|
||||
}
|
||||
|
||||
std::string& id = metadata.id;
|
||||
std::string& status = metadata.status;
|
||||
std::string& color = metadata.color;
|
||||
|
||||
const std::string::size_type rootLength = manifest.getTestRootPath().length();
|
||||
id = testPath.defaultExpectations();
|
||||
id = id.substr(rootLength + 1, id.length() - rootLength - 2);
|
||||
|
||||
bool shouldIgnore = false;
|
||||
std::string ignoreReason;
|
||||
|
||||
const std::string ignoreName = id;
|
||||
const auto it = std::find_if(ignores.cbegin(), ignores.cend(), [&ignoreName](auto pair) { return pair.first == ignoreName; });
|
||||
if (it != ignores.end()) {
|
||||
metadata.ignoredTest = shouldIgnore = true;
|
||||
ignoreReason = it->second;
|
||||
if (ignoreReason.rfind("skip", 0) == 0) {
|
||||
printf(ANSI_COLOR_GRAY "* skipped %s (%s)" ANSI_COLOR_RESET "\n", id.c_str(), ignoreReason.c_str());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (metadata.document.ObjectEmpty()) {
|
||||
metadata.metricsErrored++;
|
||||
metadata.renderErrored++;
|
||||
} else {
|
||||
runner.run(metadata);
|
||||
}
|
||||
|
||||
bool errored = metadata.metricsErrored || metadata.renderErrored || metadata.labelCutOffFound;
|
||||
bool passed = !errored && !metadata.metricsFailed && !metadata.renderFailed;
|
||||
|
||||
if (shouldIgnore) {
|
||||
if (passed) {
|
||||
status = "ignored passed";
|
||||
color = "#E8A408";
|
||||
stats.ignorePassedTests++;
|
||||
printf(ANSI_COLOR_YELLOW "* ignore %s (%s)" ANSI_COLOR_RESET "\n", id.c_str(), ignoreReason.c_str());
|
||||
} else {
|
||||
status = "ignored failed";
|
||||
color = "#9E9E9E";
|
||||
stats.ignoreFailedTests++;
|
||||
printf(ANSI_COLOR_LIGHT_GRAY "* ignore %s (%s)" ANSI_COLOR_RESET "\n", id.c_str(), ignoreReason.c_str());
|
||||
}
|
||||
} else {
|
||||
// Only fail the bots on render errors, this is a CI limitation that we need
|
||||
// to succeed on metrics failed so the rebaseline bot can run next in the
|
||||
// pipeline and collect the new baselines. The rebaseline bot will ultimately
|
||||
// report the error and block the patch from being merged.
|
||||
if (metadata.renderErrored || metadata.renderFailed) {
|
||||
returnCode = 1;
|
||||
}
|
||||
|
||||
if (passed) {
|
||||
status = "passed";
|
||||
color = "green";
|
||||
stats.passedTests++;
|
||||
printf(ANSI_COLOR_GREEN "* passed %s" ANSI_COLOR_RESET "\n", id.c_str());
|
||||
} else if (errored) {
|
||||
status = "errored";
|
||||
color = "red";
|
||||
stats.erroredTests++;
|
||||
printf(ANSI_COLOR_RED "* errored %s" ANSI_COLOR_RESET "\n", id.c_str());
|
||||
printf(ANSI_COLOR_RED "* error: %s" ANSI_COLOR_RESET "\n", metadata.errorMessage.c_str());
|
||||
} else {
|
||||
status = "failed";
|
||||
color = "red";
|
||||
stats.failedTests++;
|
||||
printf(ANSI_COLOR_RED "* failed %s" ANSI_COLOR_RESET "\n", id.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
metadatas.push_back(std::move(metadata));
|
||||
if (testStatus) {
|
||||
testStatus();
|
||||
}
|
||||
}
|
||||
|
||||
const std::string manifestName = mbgl::filesystem::path(manifestPath).stem();
|
||||
const std::string resultPath = manifest.getResultPath() + "/" + manifestName + ".html";
|
||||
std::string resultsHTML = createResultPage(stats, metadatas, shuffle, seed);
|
||||
mbgl::util::write_file(resultPath, resultsHTML);
|
||||
|
||||
const uint32_t count =
|
||||
stats.erroredTests + stats.failedTests + stats.ignoreFailedTests + stats.ignorePassedTests + stats.passedTests;
|
||||
|
||||
if (stats.passedTests) {
|
||||
printf(ANSI_COLOR_GREEN "%u passed (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.passedTests, 100.0 * stats.passedTests / count);
|
||||
}
|
||||
if (stats.ignorePassedTests) {
|
||||
printf(ANSI_COLOR_YELLOW "%u passed but were ignored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.ignorePassedTests, 100.0 * stats.ignorePassedTests / count);
|
||||
}
|
||||
if (stats.ignoreFailedTests) {
|
||||
printf(ANSI_COLOR_LIGHT_GRAY "%u ignored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.ignoreFailedTests, 100.0 * stats.ignoreFailedTests / count);
|
||||
}
|
||||
if (stats.failedTests) {
|
||||
printf(ANSI_COLOR_RED "%u failed (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.failedTests, 100.0 * stats.failedTests / count);
|
||||
}
|
||||
if (stats.erroredTests) {
|
||||
printf(ANSI_COLOR_RED "%u errored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.erroredTests, 100.0 * stats.erroredTests / count);
|
||||
}
|
||||
|
||||
printf("Results at: %s\n", mbgl::filesystem::canonical(resultPath).c_str());
|
||||
|
||||
return returnCode;
|
||||
}
|
||||
|
||||
} // namespace mbgl
|
||||
Reference in New Issue
Block a user