[core] Add filtering option to test-runner

This change adds filter command line option to render test runner,
so that test can be filtered by providing regular expression, for instance:

mbgl-render-test -f .*hillshade.*

render-test/main.cpp

@@ -3,13 +3,11 @@
 #include <mbgl/util/run_loop.hpp>
 #include <mbgl/util/io.hpp>
 
-#include "filesystem.hpp"
 #include "metadata.hpp"
 #include "parser.hpp"
 #include "runner.hpp"
 
 #include <random>
-#include <regex>
 
 #define ANSI_COLOR_RED        "\x1b[31m"
 #define ANSI_COLOR_GREEN      "\x1b[32m"
@@ -36,49 +34,18 @@ void operator delete(void* ptr, size_t) noexcept {
     AllocationIndex::deallocate(ptr);
 }
 
-namespace {
-
-TestPaths makeTestPaths(mbgl::filesystem::path stylePath) {
-    std::vector<mbgl::filesystem::path> expectations{ stylePath };
-    expectations.front().remove_filename();
-
-    const static std::regex regex{ TestRunner::getBasePath() };    
-    for (const std::string& path : TestRunner::getPlatformExpectationsPaths()) {
-        expectations.emplace_back(std::regex_replace(expectations.front().string(), regex, path));
-        assert(!expectations.back().empty());
-    }
-
-    return {
-        std::move(stylePath),
-        std::move(expectations)
-    };
-}
-
-} // namespace
-
 int main(int argc, char** argv) {
     bool recycleMap;
     bool shuffle;
     uint32_t seed;
     std::string testRootPath;
-    std::vector<std::string> ids;
+    std::vector<TestPaths> testPaths;
 
-    std::tie(recycleMap, shuffle, seed, testRootPath, ids) = parseArguments(argc, argv);
+    std::tie(recycleMap, shuffle, seed, testRootPath, testPaths) = parseArguments(argc, argv);
     const std::string::size_type rootLength = testRootPath.length();
 
     const auto ignores = parseIgnores();
 
-    // Recursively traverse through the test paths and collect test directories containing "style.json".
-    std::vector<TestPaths> testPaths;
-    testPaths.reserve(ids.size());
-    for (const auto& id : ids) {
-        for (auto& testPath : mbgl::filesystem::recursive_directory_iterator(mbgl::filesystem::path(id))) {
-            if (testPath.path().filename() == "style.json") {
-                testPaths.emplace_back(makeTestPaths(testPath));
-            }
-        }
-    }
-
     if (shuffle) {
         printf(ANSI_COLOR_YELLOW "Shuffle seed: %d" ANSI_COLOR_RESET "\n", seed);
 

render-test/parser.cpp

@@ -13,8 +13,9 @@
 #include <boost/archive/iterators/transform_width.hpp>
 #include <boost/archive/iterators/ostream_iterator.hpp>
 
-#include "parser.hpp"
+#include "filesystem.hpp"
 #include "metadata.hpp"
+#include "parser.hpp"
 #include "runner.hpp"
 
 #include <sstream>
@@ -162,6 +163,22 @@ mbgl::optional<std::string> localizeMapboxTilesetURL(const std::string& url) {
     return getIntegrationPath(url, "tilesets/", regex);
 }
 
+TestPaths makeTestPaths(mbgl::filesystem::path stylePath) {
+    std::vector<mbgl::filesystem::path> expectations{ stylePath };
+    expectations.front().remove_filename();
+
+    const static std::regex regex{ TestRunner::getBasePath() };
+    for (const std::string& path : TestRunner::getPlatformExpectationsPaths()) {
+        expectations.emplace_back(std::regex_replace(expectations.front().string(), regex, path));
+        assert(!expectations.back().empty());
+    }
+
+    return {
+        std::move(stylePath),
+        std::move(expectations)
+    };
+}
+
 } // namespace
 
 JSONReply readJson(const mbgl::filesystem::path& jsonPath) {
@@ -239,6 +256,8 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
                                         { "seed" });
     args::ValueFlag<std::string> testPathValue(argumentParser, "rootPath", "Test root rootPath",
                                                { 'p', "rootPath" });
+    args::ValueFlag<std::regex> testFilterValue(argumentParser, "filter", "Test filter regex",
+                                               { 'f', "filter" });
     args::PositionalList<std::string> testNameValues(argumentParser, "URL", "Test name(s)");
 
     try {
@@ -260,25 +279,50 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
         mbgl::Log::Info(mbgl::Event::General, stream.str());
         mbgl::Log::Error(mbgl::Event::General, e.what());
         exit(2);
+    } catch (const std::regex_error& e) {
+        mbgl::Log::Error(mbgl::Event::General, "Invalid filter regular expression: %s", e.what());
+        exit(3);
     }
 
-    const std::string testDefaultPath =
-        std::string(TEST_RUNNER_ROOT_PATH).append("/mapbox-gl-js/test/integration/render-tests");
+    mbgl::filesystem::path rootPath {testPathValue ? args::get(testPathValue) : TestRunner::getBasePath()};
+    if (!mbgl::filesystem::exists(rootPath)) {
+        mbgl::Log::Error(mbgl::Event::General, "Provided rootPath '%s' does not exist.", rootPath.string().c_str());
+        exit(4);
+    }
 
-    std::vector<std::string> ids;
+    std::vector<mbgl::filesystem::path> paths;
     for (const auto& id : args::get(testNameValues)) {
-        ids.emplace_back(TestRunner::getBasePath() + "/" + id);
+        paths.emplace_back(TestRunner::getBasePath() + "/" + id);
     }
 
-    if (ids.empty()) {
-        ids.emplace_back(TestRunner::getBasePath());
+    if (paths.empty()) {
+        paths.emplace_back(TestRunner::getBasePath());
+    }
+
+    // Recursively traverse through the test paths and collect test directories containing "style.json".
+    std::vector<TestPaths> testPaths;
+    testPaths.reserve(paths.size());
+    for (const auto& path : paths) {
+        if (!mbgl::filesystem::exists(path)) {
+            mbgl::Log::Warning(mbgl::Event::General, "Provided test folder '%s' does not exist.", path.string().c_str());
+            continue;
+        }
+        for (auto& testPath : mbgl::filesystem::recursive_directory_iterator(path)) {
+            // Skip paths that fail regexp match.
+            if (testFilterValue && !std::regex_match(testPath.path().string(), args::get(testFilterValue))) {
+                continue;
+            }
+            if (testPath.path().filename() == "style.json") {
+                testPaths.emplace_back(makeTestPaths(testPath));
+            }
+        }
     }
 
     return ArgumentsTuple {
         recycleMapFlag ? args::get(recycleMapFlag) : false,
         shuffleFlag ? args::get(shuffleFlag) : false, seedValue ? args::get(seedValue) : 1u,
-        testPathValue ? args::get(testPathValue) : testDefaultPath,
-        std::move(ids)
+        testPathValue ? args::get(testPathValue) : TestRunner::getBasePath(),
+        std::move(testPaths)
     };
 }
 

render-test/parser.hpp

@@ -12,7 +12,7 @@
 using ErrorMessage = std::string;
 using JSONReply = mbgl::variant<mbgl::JSDocument, ErrorMessage>;
 
-using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::vector<std::string>>;
+using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::vector<TestPaths>>;
 
 JSONReply readJson(const mbgl::filesystem::path&);
 std::string serializeJsonValue(const mbgl::JSValue&);