[core] Add native expression test runner

CMakeLists.txt

@@ -185,6 +185,7 @@ include(cmake/core.cmake)
 if(COMMAND mbgl_platform_test)
     include(cmake/test.cmake)
     include(cmake/render-test.cmake)
+    include(cmake/expression-test.cmake)
 endif()
 
 if(COMMAND mbgl_platform_benchmark)

cmake/expression-test.cmake

@@ -0,0 +1,27 @@
+add_executable(mbgl-expression-test
+    expression-test/main.cpp
+    expression-test/expression_test_parser.cpp
+    expression-test/expression_test_runner.cpp
+    expression-test/expression_test_logger.cpp
+)
+
+if(APPLE)
+    target_link_libraries(mbgl-expression-test PRIVATE mbgl-loop-darwin)
+else()
+    target_link_libraries(mbgl-expression-test PRIVATE mbgl-loop-uv)
+endif()
+
+target_include_directories(mbgl-expression-test
+    PRIVATE src
+    PRIVATE expression-test
+    PRIVATE render-test
+)
+
+target_link_libraries(mbgl-expression-test PRIVATE
+    mbgl-core
+    Mapbox::Base::Extras::args
+    Mapbox::Base::Extras::filesystem
+    Mapbox::Base::Extras::rapidjson
+)
+
+add_definitions(-DTEST_RUNNER_ROOT_PATH="${CMAKE_SOURCE_DIR}")

expression-test/expression_test_logger.cpp

@@ -0,0 +1,177 @@
+#include "expression_test_logger.hpp"
+#include "expression_test_runner.hpp"
+#include "filesystem.hpp"
+
+#include <mbgl/util/io.hpp>
+#include <mbgl/util/string.hpp>
+
+#include <sstream>
+
+using namespace mbgl;
+using namespace std::literals;
+
+namespace {
+
+const char* resultsStyle = R"HTML(
+<style>
+    body { font: 18px/1.2 -apple-system, BlinkMacSystemFont, "Helvetica Neue", Helvetica, Arial, sans-serif; padding: 10px; }
+    h1 { font-size: 32px; margin-bottom: 0; }
+    button { vertical-align: middle; }
+    h2 { font-size: 24px; font-weight: normal; margin: 10px 0 10px; line-height: 1; }
+    .test { border-bottom: 1px dotted #bbb; padding-bottom: 5px; }
+    .tests { border-top: 1px dotted #bbb; margin-top: 10px; }
+    .test p, .test pre { margin: 0 0 10px; }
+    .test pre { font-size: 14px; }
+    .label { color: white; font-size: 18px; padding: 2px 6px 3px; border-radius: 3px; margin-right: 3px; vertical-align: bottom; display: inline-block; }
+    .hide { display: none; }
+    .test.failed > h2 > span { background: red; }
+    .test.passed > h2 > span { background: green; }
+    .test.ignored > h2 > span { background: grey; }
+    .test.errored > h2 > span { background: #ff006f; }
+    .test.ignored.passed > h2 > span { background: #E8A408; }
+</style>
+)HTML";
+
+const char* resultsHeaderButtons = R"HTML(
+    <button id='toggle-sequence'>Toggle test sequence</button>
+    <button id='toggle-passed'>Toggle passed tests</button>
+    <button id='toggle-ignored'>Toggle ignored tests</button>
+</h1>
+)HTML";
+
+const char* resultsScript = R"HTML(
+<script>
+document.getElementById('toggle-passed').addEventListener('click', function (e) {
+    for (const row of document.querySelectorAll('.test.passed')) {
+        row.classList.toggle('hide');
+    }
+});
+document.getElementById('toggle-ignored').addEventListener('click', function (e) {
+    for (const row of document.querySelectorAll('.test.ignored')) {
+        row.classList.toggle('hide');
+    }
+});
+document.getElementById('toggle-sequence').addEventListener('click', function (e) {
+    document.getElementById('test-sequence').classList.toggle('hide');
+});
+</script>
+)HTML";
+
+std::string createResultItem(const TestRunOutput& result, const std::string& status, bool shouldHide) {
+    std::ostringstream html;
+    html << "<div class=\"test " << status << (shouldHide ? " hide" : "") << "\">\n";
+    html << R"(<h2><span class="label">)"  << status << "</span> " << result.id << "</h2>\n";
+
+    html << "<p><pre>"s << result.expression << "</pre></p>\n"s;
+    if (result.passed) {
+        html << "<strong>Serialized:</strong><p><pre>"s << result.serialized << "</pre></p>\n"s;
+    } else {
+        html << "<p><strong>Difference:</strong><pre>" << result.text << "</pre></p>\n";
+    }
+    html << "</div>\n";
+
+    return html.str();
+}
+
+std::string createResultPage(const TestStats& stats, bool shuffle, uint32_t seed) {
+    const std::size_t unsuccessfulCount = stats.errored.size() + stats.failed.size();
+    const bool unsuccessful = unsuccessfulCount > 0;
+    std::ostringstream resultsPage;
+
+    // Style
+    resultsPage << resultsStyle;
+
+    // Header with buttons
+    if (unsuccessful) {
+        resultsPage << R"HTML(<h1 style="color: red;">)HTML";
+        resultsPage << util::toString(unsuccessfulCount) << " tests failed.";
+    } else {
+        resultsPage << R"HTML(<h1 style="color: green;">)HTML";
+        resultsPage << "All tests passed!";
+    }
+
+    resultsPage << resultsHeaderButtons;
+
+    // Test sequence
+    {
+        resultsPage << "<div id='test-sequence' class='hide'>\n";
+
+        // Failed tests
+        if (unsuccessful) {
+            resultsPage  << "<p><strong>Failed tests:</strong>";
+            for (const auto& failed : stats.failed) {
+                resultsPage << failed.id << " ";
+            }
+            resultsPage  << "<p><strong>Errored tests:</strong>";
+            for (const auto& errored : stats.errored) {
+                resultsPage << errored.id << " ";
+            }
+            resultsPage << "</p>\n";
+        }
+
+        // Test sequence
+        resultsPage << "<p><strong>Test sequence: </strong>";
+        for (const auto& id : stats.ids) {
+            resultsPage << id << " ";
+        }
+        resultsPage << "</p>\n";
+
+        // Shuffle
+        if (shuffle) {
+            resultsPage << "<p><strong>Shuffle seed</strong>: " << util::toString(seed) << "</p>\n";
+        }
+
+        resultsPage << "</div>\n";
+    }
+
+    // Script
+    resultsPage << resultsScript;
+
+     // Tests
+    resultsPage << "<div class=\"tests\">\n";
+    const auto appendResult = [&] (const auto& results, const std::string& status, bool hide = false) {
+        for (const auto& result : results) {
+            resultsPage << createResultItem(result, status, hide);
+        }
+    };
+
+    appendResult(stats.passed, "passed"s, unsuccessful);
+    appendResult(stats.failed, "failed"s);
+    appendResult(stats.errored, "errored"s);
+    appendResult(stats.ignorePassed, "ignored passed"s, unsuccessful);
+    appendResult(stats.ignoreFailed, "ignored"s, true);
+    resultsPage << "</div>\n";
+
+    return resultsPage.str();
+}
+
+} // namespace
+
+void printStats(const TestStats& stats) {
+    const std::size_t count = stats.testCount();
+    if (std::size_t passedTests = stats.passed.size()) {
+        printf(ANSI_COLOR_GREEN "%zu passed (%.1lf%%)" ANSI_COLOR_RESET "\n", passedTests, 100.0 * passedTests / count);
+    }
+    if (std::size_t ignorePassedTests = stats.ignorePassed.size()) {
+        printf(ANSI_COLOR_YELLOW "%zu passed but were ignored (%.1lf%%)" ANSI_COLOR_RESET "\n", ignorePassedTests, 100.0 * ignorePassedTests / count);
+    }
+    if (std::size_t ignoreFailedTests = stats.ignoreFailed.size()) {
+        printf(ANSI_COLOR_LIGHT_GRAY "%zu ignored (%.1lf%%)" ANSI_COLOR_RESET "\n", ignoreFailedTests, 100.0 * ignoreFailedTests / count);
+    }
+    if (std::size_t failedTests = stats.failed.size()) {
+        printf(ANSI_COLOR_RED "%zu failed (%.1lf%%)" ANSI_COLOR_RESET "\n", failedTests, 100.0 * failedTests / count);
+    }
+    if (std::size_t erroredTests = stats.errored.size()) {
+        printf(ANSI_COLOR_RED "%zu errored (%.1lf%%)" ANSI_COLOR_RESET "\n", erroredTests, 100.0 * erroredTests / count);
+    }
+}
+
+void writeHTMLResults(const TestStats& stats, const std::string& rootPath, bool shuffle, uint32_t seed) {
+    filesystem::path path = filesystem::path(rootPath) / "index.html"s;
+    try {
+        util::write_file(path.string(), createResultPage(stats, shuffle, seed));
+        printf("Results at: %s\n", path.string().c_str());
+    } catch (std::exception&) {
+        printf(ANSI_COLOR_RED "* ERROR can't write result page %s" ANSI_COLOR_RESET "\n", path.string().c_str());
+    }
+}

expression-test/expression_test_logger.hpp

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <string>
+
+#define ANSI_COLOR_RED        "\x1b[31m"
+#define ANSI_COLOR_GREEN      "\x1b[32m"
+#define ANSI_COLOR_YELLOW     "\x1b[33m"
+#define ANSI_COLOR_LIGHT_GRAY "\x1b[90m"
+#define ANSI_COLOR_RESET      "\x1b[0m"
+
+class TestStats;
+
+void printStats(const TestStats&);
+void writeHTMLResults(const TestStats&, const std::string&, bool, uint32_t);

expression-test/expression_test_parser.cpp

@@ -0,0 +1,536 @@
+#include "expression_test_parser.hpp"
+
+#include <mbgl/util/io.hpp>
+#include <mbgl/util/logging.hpp>
+#include <mbgl/util/variant.hpp>
+#include <mbgl/style/conversion/function.hpp>
+#include <mbgl/style/rapidjson_conversion.hpp>
+#include <mbgl/style/expression/parsing_context.hpp>
+
+#include <mapbox/geojson/rapidjson.hpp>
+#include <rapidjson/writer.h>
+#include <rapidjson/prettywriter.h>
+#include <rapidjson/stringbuffer.h>
+
+#include <args.hxx>
+
+using namespace mbgl;
+using namespace mbgl::style;
+using namespace mbgl::style::conversion;
+using namespace std::literals;
+
+namespace {
+
+void writeJSON(rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer, const Value& value) {
+    value.match(
+        [&] (const NullValue&) { writer.Null(); },
+        [&] (bool b) { writer.Bool(b); },
+        [&] (uint64_t u) { writer.Uint64(u); },
+        [&] (int64_t i) { writer.Int64(i); },
+        [&] (double d) { d == std::floor(d) ? writer.Int64(d) : writer.Double(d); },
+        [&] (const std::string& s) { writer.String(s); },
+        [&] (const std::vector<Value>& arr) {
+            writer.StartArray();
+            for(const auto& item : arr) {
+                writeJSON(writer, item);
+            }
+            writer.EndArray();
+        },
+        [&] (const std::unordered_map<std::string, Value>& obj) {
+            writer.StartObject();
+            for(const auto& entry : obj) {
+                writer.Key(entry.first.c_str());
+                writeJSON(writer, entry.second);
+            }
+            writer.EndObject();
+        }
+    );
+}
+
+using ErrorMessage = std::string;
+using JSONReply = variant<JSDocument, ErrorMessage>;
+JSONReply readJson(const filesystem::path& jsonPath) {
+    auto maybeJSON = util::readFile(jsonPath);
+    if (!maybeJSON) {
+        return { "Unable to open file "s + jsonPath.string() };
+    }
+
+    JSDocument document;
+    document.Parse<rapidjson::kParseFullPrecisionFlag>(*maybeJSON);
+    if (document.HasParseError()) {
+        return { formatJSONParseError(document) };
+    }
+
+    return { std::move(document) };
+}
+
+std::string toString(const JSValue& value) {
+    assert(value.IsString());
+    return { value.GetString(), value.GetStringLength() };
+}
+
+optional<Value> toValue(const JSValue& jsvalue) {
+    if (jsvalue.IsNull()) {
+        return Value{};
+    }
+
+    if (jsvalue.IsArray()) {
+        std::vector<Value> values;
+        values.reserve(jsvalue.GetArray().Size());
+        for (const auto& v : jsvalue.GetArray()) {
+            if (auto value = toValue(v)) {
+                values.emplace_back(std::move(*value));
+            }
+        }
+        return {std::move(values)};
+    }
+
+    if (jsvalue.IsObject()) {
+        std::unordered_map<std::string, Value> value_map;
+        for (const auto& pair : jsvalue.GetObject()) {
+            if (auto value = toValue(pair.value)) {
+                value_map.emplace(toString(pair.name), std::move(*value));
+            }
+        }
+        return {std::move(value_map)};
+    }
+
+    if (!jsvalue.IsArray() && !jsvalue.IsObject()) {
+        return toValue(Convertible(&jsvalue));
+    }
+
+    return nullopt;
+}
+
+style::expression::type::Type stringToType(const std::string& type) {
+    using namespace style::expression;
+    if (type == "string"s || type == "number-format"s) {
+        return type::String;
+    } else if (type == "number"s) {
+        return type::Number;
+    } else if (type == "boolean"s) {
+        return type::Boolean;
+    } else if (type == "object"s) {
+        return type::Object;
+    } else if (type == "color"s) {
+        return type::Color;
+    } else if (type == "value"s) {
+        return type::Value;
+    } else if (type == "formatted"s) {
+        return type::Formatted;
+    }
+
+    // Should not reach.
+    assert(false);
+    return type::Null;
+}
+
+optional<style::expression::type::Type> toExpressionType(const PropertySpec& spec) {
+    using namespace style::expression;
+    if (spec.type == "array") {
+        type::Type itemType = spec.value.empty() ? type::Value : stringToType(spec.value);
+        if (spec.length) {
+            return {type::Array(itemType, spec.length)};
+        }
+        return {type::Array(itemType)};
+    }
+
+    if (spec.type == "enum") {
+        return {type::String};
+    }
+
+    return spec.type.empty() ? nullopt : optional<type::Type>{stringToType(spec.type)};
+}
+
+void parseCompiled(const JSValue& compiledValue, TestData& data) {
+    const auto& compiled = compiledValue.GetObject();
+    assert(compiled.HasMember("result"));
+    assert(compiled["result"].IsString());
+    const std::string& result = toString(compiled["result"]);
+    data.expected.compiled.success = result == "success";
+
+    if (compiled.HasMember("isFeatureConstant")) {
+        assert(compiled["isFeatureConstant"].IsBool());
+        data.expected.compiled.isFeatureConstant = compiled["isFeatureConstant"].GetBool();
+    }
+
+    if (compiled.HasMember("isZoomConstant")) {
+        assert(compiled["isZoomConstant"].IsBool());
+        data.expected.compiled.isZoomConstant = compiled["isZoomConstant"].GetBool();
+    }
+
+    if (compiled.HasMember("type")) {
+        assert(compiled["type"].IsString());
+        data.expected.compiled.serializedType = toString(compiled["type"]);
+    }
+
+    if (compiled.HasMember("errors")) {
+        assert(compiled["errors"].IsArray());
+        for (const auto& errorVal : compiled["errors"].GetArray()) {
+            assert(errorVal.IsObject());
+            const auto& errorObject = errorVal.GetObject();
+            assert(errorObject.HasMember("key"));
+            assert(errorObject.HasMember("error"));
+
+            std::unordered_map<std::string, Value> errorMap;
+            errorMap.emplace("key"s, Value{toString(errorObject["key"])});
+            errorMap.emplace("error"s, Value{toString(errorObject["error"])});
+            data.expected.compiled.errors.emplace_back(Value{std::move(errorMap)});
+        }
+    }
+}
+
+void parseExpected(const JSValue& expectedValue, TestData& data) {
+    assert(expectedValue.IsObject());
+    const auto& expected = expectedValue.GetObject();
+    assert(expected.HasMember("compiled"));
+    parseCompiled(expected["compiled"], data);
+
+    // set outputs
+    if (expected.HasMember("outputs")) {
+        data.expected.outputs = toValue(expected["outputs"]);
+    }
+
+    // set serialized
+    if (expected.HasMember("serialized")) {
+        data.expected.serialized = toValue(expected["serialized"]);
+    }
+}
+
+void parsePropertySpec(const JSValue& value, TestData& data) {
+    const auto& propertySpec = value.GetObject();
+    PropertySpec spec;
+
+    if (propertySpec.HasMember("type")) {
+        assert(propertySpec["type"].IsString());
+        spec.type = toString(propertySpec["type"]);
+    }
+
+    if (propertySpec.HasMember("value")) {
+        assert(propertySpec["value"].IsString());
+        spec.value = toString(propertySpec["value"]);
+    }
+
+    if (propertySpec.HasMember("length")) {
+        assert(propertySpec["length"].IsNumber());
+        spec.length = propertySpec["length"].GetDouble();
+    }
+
+    if (propertySpec.HasMember("property-type")) {
+        assert(propertySpec["property-type"].IsString());
+        spec.isDataDriven = true;
+    }
+
+    if (propertySpec.HasMember("expression")) {
+        assert(propertySpec["expression"].IsObject());
+        spec.expression = toValue(propertySpec["expression"]);
+    }
+
+    data.spec = std::move(spec);
+}
+
+bool parseInputs(const JSValue& inputsValue, TestData& data) {
+    assert(inputsValue.IsArray());
+    for (const auto& input : inputsValue.GetArray()) {
+        assert(input.IsArray());
+        assert(input.Size() == 2);
+        assert(input[0].IsObject());
+        assert(input[1].IsObject());
+
+        // Parse evaluation context, zoom.
+        optional<float> zoom;
+        const auto& evaluationContext = input[0].GetObject();
+        if (evaluationContext.HasMember("zoom")) {
+            assert(evaluationContext["zoom"].IsNumber());
+            zoom = evaluationContext["zoom"].GetDouble();
+        }
+
+        // Parse heatmap density
+        optional<double> heatmapDensity;
+        if (evaluationContext.HasMember("heatmapDensity")) {
+            assert(evaluationContext["heatmapDensity"].IsNumber());
+            heatmapDensity = evaluationContext["heatmapDensity"].GetDouble();
+        }
+
+        // Parse feature properties
+        Feature feature(mapbox::geometry::point<double>(0.0, 0.0));
+        const auto& featureObject = input[1].GetObject();
+        if (featureObject.HasMember("properties")) {
+            assert(featureObject["properties"].IsObject());
+            feature.properties = mapbox::geojson::convert<PropertyMap>(featureObject["properties"]);
+        }
+
+        if (featureObject.HasMember("geometry")) {
+            assert(featureObject["geometry"].IsObject());
+            feature.geometry = mapbox::geojson::convert<mapbox::geometry::geometry<double>>(featureObject["geometry"]);
+        }
+
+        if (featureObject.HasMember("id")) {
+            assert(featureObject["id"].IsNumber() || featureObject["id"].IsString());
+            feature.id = mapbox::geojson::convert<mapbox::feature::identifier>(featureObject["id"]);
+        }
+
+        data.inputs.emplace_back(std::move(zoom), std::move(heatmapDensity), std::move(feature));
+    }
+    return true;
+}
+
+} // namespace
+
+std::tuple<filesystem::path, std::vector<filesystem::path>, bool, uint32_t> parseArguments(int argc, char** argv) {
+    args::ArgumentParser argumentParser("Mapbox GL Expression Test Runner");
+
+    args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", { 'h', "help" });
+    args::Flag shuffleFlag(argumentParser, "shuffle", "Toggle shuffling the tests order",
+                           { 's', "shuffle" });
+    args::ValueFlag<uint32_t> seedValue(argumentParser, "seed", "Shuffle seed (default: random)",
+                                        { "seed" });
+    args::PositionalList<std::string> testNameValues(argumentParser, "URL", "Test name(s)");
+
+    try {
+        argumentParser.ParseCLI(argc, argv);
+    } catch (const args::Help&) {
+        std::ostringstream stream;
+        stream << argumentParser;
+        Log::Info(Event::General, stream.str());
+        exit(0);
+    } catch (const args::ParseError& e) {
+        std::ostringstream stream;
+        stream << argumentParser;
+        Log::Info(Event::General, stream.str());
+        Log::Error(Event::General, e.what());
+        exit(1);
+    } catch (const args::ValidationError& e) {
+        std::ostringstream stream;
+        stream << argumentParser;
+        Log::Info(Event::General, stream.str());
+        Log::Error(Event::General, e.what());
+        exit(2);
+    }
+
+    filesystem::path rootPath {std::string(TEST_RUNNER_ROOT_PATH).append("/mapbox-gl-js/test/integration/expression-tests")};
+    if (!filesystem::exists(rootPath)) {
+        Log::Error(Event::General, "Test path '%s' does not exist.", rootPath.string().c_str());
+        exit(3);
+    }
+
+    std::vector<filesystem::path> paths;
+    for (const auto& testName : args::get(testNameValues)) {
+        paths.emplace_back(rootPath.string() + "/" + testName);
+    }
+
+    if (paths.empty()) {
+        paths.emplace_back(rootPath);
+    }
+
+    // Recursively traverse through the test paths and collect test directories containing "test.json".
+    std::vector<filesystem::path> testPaths;
+    testPaths.reserve(paths.size());
+    for (const auto& path : paths) {
+        if (!filesystem::exists(path)) {
+            Log::Warning(Event::General, "Provided test folder '%s' does not exist.", path.string().c_str());
+            continue;
+        }
+
+        for (auto& testPath : filesystem::recursive_directory_iterator(path)) {
+            if (testPath.path().filename() == "test.json") {
+                testPaths.emplace_back(testPath.path());
+            }
+        }
+    }
+
+    return Arguments{ std::move(rootPath),
+                      std::move(testPaths),
+                      shuffleFlag ? args::get(shuffleFlag) : false,
+                      seedValue ? args::get(seedValue) : 1u };
+}
+
+Ignores parseExpressionIgnores() {
+    Ignores ignores;
+    const auto mainIgnoresPath = filesystem::path(TEST_RUNNER_ROOT_PATH).append("platform/node/test/ignores.json");
+    auto maybeIgnores = readJson(mainIgnoresPath);
+    if (!maybeIgnores.is<JSDocument>()) { // NOLINT
+        return {};
+    }
+
+    for (const auto& property : maybeIgnores.get<JSDocument>().GetObject()) {
+        std::string id{ toString(property.name) };
+        // Keep only expression-test ignores
+        if (id.rfind("expression-tests", 0) != 0) {
+            continue;
+        }
+        std::string reason{ toString(property.value) };
+        ignores.emplace_back(std::move(id), std::move(reason));
+    }
+
+    return ignores;
+}
+
+optional<TestData> parseTestData(const filesystem::path& path) {
+    TestData data;
+    auto maybeJson = readJson(path.string());
+    if (!maybeJson.is<JSDocument>()) { // NOLINT
+        Log::Error(Event::General, "Cannot parse test '%s'.", path.string().c_str());
+        return nullopt;
+    }
+
+    data.document = std::move(maybeJson.get<JSDocument>());
+
+    // Check that mandatory test data members are present.
+    if (!data.document.HasMember("expression") || !data.document.HasMember("expected")) {
+        Log::Error(Event::General, "Test fixture '%s' does not contain required data.", path.string().c_str());
+        return nullopt;
+    }
+
+    // Parse propertySpec
+    if (data.document.HasMember("propertySpec")) {
+        assert(data.document["propertySpec"].IsObject());
+        parsePropertySpec(data.document["propertySpec"], data);
+    }
+
+    // Parse expected
+    parseExpected(data.document["expected"], data);
+
+    // Parse inputs
+    if (data.document.HasMember("inputs") && !parseInputs(data.document["inputs"], data)) {
+        Log::Error(Event::General,"Can't convert inputs value for '%s'", path.string().c_str());
+        return nullopt;
+    }
+
+    return {std::move(data)};
+}
+
+std::string toJSON(const Value& value, unsigned indent, bool singleLine) {
+    rapidjson::StringBuffer buffer;
+    rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
+    if (singleLine) {
+        writer.SetFormatOptions(rapidjson::kFormatSingleLineArray);
+    }
+    writer.SetIndent(' ', indent);
+    writeJSON(writer, value);
+    return buffer.GetString();
+}
+
+JSDocument toDocument(const Value& value) {
+    JSDocument document;
+    document.Parse<rapidjson::kParseFullPrecisionFlag>(toJSON(value));
+    return document;
+}
+
+Value toValue(const Compiled& compiled) {
+    std::unordered_map<std::string, Value> compiledObject;
+    compiledObject.emplace("result", compiled.success ? "success"s : "error"s);
+    if (compiled.success) {
+        compiledObject.emplace("isFeatureConstant", compiled.isFeatureConstant);
+        compiledObject.emplace("isZoomConstant", compiled.isZoomConstant);
+        compiledObject.emplace("type", compiled.serializedType);
+    } else {
+        compiledObject.emplace("errors", compiled.errors);
+    }
+    return {std::move(compiledObject)};
+}
+
+// Serializes native expression types to JS format.
+// Color and Formatted are exceptions.
+// TODO: harmonize serialized format to remove this conversion.
+optional<Value> toValue(const expression::Value& exprValue) {
+    return exprValue.match(
+        [](const Color& c) -> optional<Value> {
+            std::vector<Value> color { double(c.r), double(c.g), double(c.b), double(c.a) };
+            return {Value{std::move(color)}};
+        },
+        [](const expression::Formatted& formatted) -> optional<Value> {
+            std::unordered_map<std::string, Value> serialized;
+            std::vector<Value> sections;
+            for (const auto& section : formatted.sections) {
+                std::unordered_map<std::string, Value> serializedSection;
+                serializedSection.emplace("text", section.text);
+                if (section.fontScale) {
+                    serializedSection.emplace("scale", *section.fontScale);
+                } else {
+                    serializedSection.emplace("scale", NullValue());
+                }
+                if (section.fontStack) {
+                    std::string fontStackString;
+                    serializedSection.emplace("fontStack", fontStackToString(*section.fontStack));
+                } else {
+                    serializedSection.emplace("fontStack", NullValue());
+                }
+                if (section.textColor) {
+                    serializedSection.emplace("textColor", section.textColor->toObject());
+                } else {
+                    serializedSection.emplace("textColor", NullValue());
+                }
+                sections.emplace_back(serializedSection);
+            }
+            serialized.emplace("sections", sections);
+            return {Value{std::move(serialized)}};
+        },
+        [](const std::vector<expression::Value>& values) -> optional<Value> {
+            std::vector<Value> mbglValues;
+            for (const auto& value : values) {
+                if (auto converted = expression::fromExpressionValue<Value>(value)) {
+                    mbglValues.emplace_back(std::move(*converted));
+                }
+            }
+            return {Value{std::move(mbglValues)}};
+        },
+        [](const std::unordered_map<std::string, expression::Value>& valueMap) -> optional<Value> {
+            std::unordered_map<std::string, Value> mbglValueMap;
+            for (const auto& pair : valueMap) {
+                if (auto converted = expression::fromExpressionValue<Value>(pair.second)) {
+                    mbglValueMap.emplace(pair.first, std::move(*converted));
+                }
+            }
+            return {Value{std::move(mbglValueMap)}};
+        },
+        [](const auto& v) { return expression::fromExpressionValue<Value>(v); });
+}
+
+std::unique_ptr<style::expression::Expression> parseExpression(const JSValue& value,
+                                                               optional<PropertySpec>& spec,
+                                                               TestResult& result) {
+    optional<style::expression::type::Type> expected = spec ? toExpressionType(*spec) : nullopt;
+    expression::ParsingContext ctx = expected ? expression::ParsingContext(*expected) :
+                                                expression::ParsingContext();
+    Convertible convertible(&value);
+    expression::ParseResult parsed;
+    if (value.IsObject() && !value.IsArray() && expected){
+        Error error;
+        parsed = convertFunctionToExpression(*expected, convertible, error, false /*convert tokens*/);
+        if (!parsed) {
+            // TODO: should the error message be checked for function conversion?
+        }
+    } else {
+        parsed = ctx.parseLayerPropertyExpression(convertible);
+        if (!parsed) {
+            for (const auto& parsingError : ctx.getErrors()) {
+                std::unordered_map<std::string, Value> errorMap;
+                errorMap.emplace("key"s, Value{parsingError.key});
+                errorMap.emplace("error"s, Value{parsingError.message});
+                result.compiled.errors.emplace_back(Value{std::move(errorMap)});
+            }
+        }
+    }
+
+    result.expression = toValue(value);
+    result.compiled.success = bool(parsed);
+    if (parsed) {
+        result.compiled.isFeatureConstant = expression::isFeatureConstant(**parsed);
+        result.compiled.isZoomConstant = expression::isZoomConstant(**parsed);
+        result.compiled.serializedType = style::expression::type::toString((*parsed)->getType());
+        result.serialized = (*parsed)->serialize();
+        return std::move(*parsed);
+    }
+
+    return nullptr;
+}
+
+std::unique_ptr<style::expression::Expression> parseExpression(const optional<Value>& value,
+                                                               optional<PropertySpec>& spec,
+                                                               TestResult& result) {
+    assert(value);
+    auto document = toDocument(*value);
+    assert(!document.HasParseError());
+    return parseExpression(document, spec, result);
+}

expression-test/expression_test_parser.hpp

@@ -0,0 +1,94 @@
+#pragma once
+
+#include "filesystem.hpp"
+
+#include <mbgl/style/expression/expression.hpp>
+#include <mbgl/util/feature.hpp>
+#include <mbgl/util/optional.hpp>
+#include <mbgl/util/rapidjson.hpp>
+
+#include <vector>
+#include <string>
+
+using namespace mbgl;
+
+struct Input {
+    Input(optional<float> zoom_, optional<double> heatmapDensity_, Feature feature_)
+        : zoom(std::move(zoom_)),
+          heatmapDensity(std::move(heatmapDensity_)),
+          feature(std::move(feature_)) {}
+    optional<float> zoom;
+    optional<double> heatmapDensity;
+    Feature feature;
+};
+
+struct Compiled {
+    bool operator==(const Compiled& other) const {
+        bool typeEqual = success == other.success &&
+                         isFeatureConstant == other.isFeatureConstant &&
+                         isZoomConstant == other.isZoomConstant &&
+                         serializedType == other.serializedType &&
+                         errors == other.errors;
+        return typeEqual;
+    }
+
+    bool success = false;
+    bool isFeatureConstant = false;
+    bool isZoomConstant = false;
+    std::string serializedType;
+    std::vector<Value> errors;
+};
+
+struct TestResult {
+    Compiled compiled;
+    optional<Value> expression;
+    optional<Value> outputs;
+    optional<Value> serialized;
+};
+
+struct PropertySpec {
+    std::string type;
+    std::string value;
+    std::size_t length = 0;
+    bool isDataDriven = false;
+    optional<Value> expression;
+};
+
+class TestData {
+public:
+    std::vector<Input> inputs;
+    TestResult expected;
+    TestResult result;
+    TestResult recompiled;
+    optional<PropertySpec> spec;
+    JSDocument document;
+};
+
+struct Ignore {
+    Ignore(std::string id_, std::string reason_)
+        : id(std::move(id_)),
+          reason(std::move(reason_)) {}
+
+    std::string id;
+    std::string reason;
+};
+
+using Arguments = std::tuple<filesystem::path, std::vector<filesystem::path>, bool, uint32_t>;
+Arguments parseArguments(int argc, char** argv);
+
+using Ignores = std::vector<Ignore>;
+Ignores parseExpressionIgnores();
+optional<TestData> parseTestData(const filesystem::path&);
+
+std::string toJSON(const Value& value, unsigned indent = 0, bool singleLine = false);
+JSDocument toDocument(const Value&);
+Value toValue(const Compiled&);
+optional<Value> toValue(const style::expression::Value&);
+
+std::unique_ptr<style::expression::Expression> parseExpression(const JSValue&,
+                                                               optional<PropertySpec>&,
+                                                               TestResult&);
+std::unique_ptr<style::expression::Expression> parseExpression(const optional<Value>&,
+                                                               optional<PropertySpec>&,
+                                                               TestResult&);
+

expression-test/expression_test_runner.cpp

@@ -0,0 +1,303 @@
+#include "expression_test_runner.hpp"
+#include "expression_test_parser.hpp"
+#include "expression_test_logger.hpp"
+#include "filesystem.hpp"
+
+#include <mbgl/util/io.hpp>
+
+#include <rapidjson/writer.h>
+#include <rapidjson/prettywriter.h>
+#include <rapidjson/stringbuffer.h>
+
+#include <sstream>
+#include <regex>
+
+using namespace std::literals;
+
+namespace {
+
+// Strip precision for numbers, so that we can compare evaluated results with fixtures.
+// Copied from JS expression harness.
+Value stripPrecision(const Value& value) {
+    const double decimalSigFigs = 6;
+    if (auto num = numericValue<double>(value)) {
+        if (*num == 0) {
+            return *num;
+        }
+
+        const double multiplier = std::pow(10,
+            std::max(0.0, decimalSigFigs - std::ceil(std::log10(std::fabs(*num)))));
+
+        // We strip precision twice in a row here to avoid cases where
+        // stripping an already stripped number will modify its value
+        // due to bad floating point precision luck
+        // eg `Math.floor(8.16598 * 100000) / 100000` -> 8.16597
+        const double firstStrip = std::floor(*num * multiplier) / multiplier;
+        return std::floor(firstStrip * multiplier) / multiplier;
+    }
+
+    if (value.is<std::vector<Value>>()) {
+        std::vector<Value> stripped;
+        const auto& vec = value.get<std::vector<Value>>();
+        stripped.reserve(vec.size());
+        for (const auto& val : vec) {
+            stripped.emplace_back(stripPrecision(val));
+        }
+        return stripped;
+    } else if (value.is<std::unordered_map<std::string, Value>>()) {
+        std::unordered_map<std::string, Value> stripped;
+        const auto& map = value.get<std::unordered_map<std::string, Value>>();
+        for (const auto& pair : map) {
+            stripped.emplace(pair.first, stripPrecision(pair.second));
+        }
+        return stripped;
+    }
+
+    return value;
+}
+
+bool deepEqual(const Value& a, const Value& b) {
+    const auto& anum = numericValue<double>(a);
+    const auto& bnum = numericValue<double>(b);
+    if (anum && bnum) {
+        return stripPrecision(*anum) == stripPrecision(*bnum);
+    }
+
+    if (a.which() != b.which()) {
+        return false;
+    }
+
+    if (a.is<std::vector<Value>>()) {
+        const auto& avec = a.get<std::vector<Value>>();
+        const auto& bvec = b.get<std::vector<Value>>();
+        if (avec.size() != bvec.size()) {
+            return false;
+        }
+        for (std::size_t i = 0; i < avec.size(); ++i) {
+            if (!deepEqual(avec[i], bvec[i])) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    if (a.is<std::unordered_map<std::string, Value>>()) {
+        const auto& amap = a.get<std::unordered_map<std::string, Value>>();
+        const auto& bmap = b.get<std::unordered_map<std::string, Value>>();
+        if (amap.size() != bmap.size()) {
+            return false;
+        }
+        for (const auto& pair : amap) {
+            auto it = bmap.find(pair.first);
+            if (it == bmap.end()) {
+                return false;
+            }
+            if (!deepEqual(pair.second, it->second)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    return a == b;
+}
+
+bool deepEqual(const optional<Value>& a, const optional<Value>& b) {
+    if ((a && !b) || (!a && b)) {
+        return false;
+    }
+
+    if (a && b) {
+        return deepEqual(*a, *b);
+    }
+
+    return true;
+}
+
+std::vector<std::string> tokenize(std::string str) {
+    std::vector<std::string> tokens;
+    std::regex re("\n");
+    std::copy(std::regex_token_iterator<std::string::iterator>(str.begin(), str.end(), re, -1),
+              std::regex_token_iterator<std::string::iterator>(),
+              std::back_inserter(tokens));
+    return tokens;
+}
+
+std::string simpleDiff(const Value& result, const Value& expected) {
+    std::vector<std::string> resultTokens {tokenize(toJSON(result, 2, true))};
+    std::vector<std::string> expectedTokens {tokenize(toJSON(expected, 2, true))};
+    std::size_t maxLength = std::max(resultTokens.size(), expectedTokens.size());
+    std::ostringstream diff;
+    const auto flush = [] (const std::vector<std::string>& vec, std::size_t pos, std::ostringstream& out, std::string separator) {
+        for (std::size_t j = pos; j < vec.size(); ++j) {
+            out << separator << vec[j] << std::endl;
+        }
+    };
+
+    for (std::size_t i = 0; i < maxLength; ++i) {
+        if (resultTokens.size() <= i) {
+            flush(expectedTokens, i, diff, "+"s);
+            break;
+        }
+
+        if (expectedTokens.size() <= i) {
+            flush(expectedTokens, i, diff, "-"s);
+            break;
+        }
+
+        if (resultTokens[i] != expectedTokens[i]) {
+            diff << "-"s << expectedTokens[i] << std::endl;
+            diff << "+"s << resultTokens[i] << std::endl;
+        } else {
+            diff << resultTokens[i] << std::endl;
+        }
+    }
+    return diff.str();
+}
+
+void rewriteRoundtrippedType(const std::string& expected, std::string& actual) {
+    if (!expected.rfind("array", 0) && !actual.rfind("array", 0)) {
+        actual = expected;
+    }
+}
+
+void updateTest(TestData& data) {
+    assert(data.document["expected"].IsObject());
+    auto& expected = data.document["expected"];
+    auto compiled = toDocument(toValue(data.result.compiled));
+    assert(!compiled.HasParseError());
+    expected["compiled"].Swap(compiled);
+
+    if (data.result.outputs) {
+        auto outputs = toDocument(stripPrecision(data.result.outputs.value_or(Value{})));
+        assert(!outputs.HasParseError());
+        expected["outputs"].Swap(outputs);
+    } else {
+        expected.RemoveMember("outputs");
+    }
+
+    if (data.result.serialized) {
+        auto serialized = toDocument(*data.result.serialized);
+        assert(!serialized.HasParseError());
+        expected["serialized"].Swap(serialized);
+    } else {
+        expected.RemoveMember("serialized");
+    }
+}
+
+void writeTestData(const JSDocument& document, const std::string& rootPath, const std::string& id) {
+    rapidjson::StringBuffer buffer;
+    rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
+    writer.SetFormatOptions(rapidjson::kFormatSingleLineArray);
+    writer.SetIndent(' ', 2);
+    document.Accept(writer);
+    buffer.Put('\n');
+    filesystem::path path = filesystem::path(rootPath) / id / "test.json"s;
+    try {
+        util::write_file(path.string(), {buffer.GetString(), buffer.GetSize()});
+    } catch (std::exception&) {
+        printf(ANSI_COLOR_RED "* ERROR can't update '%s' test" ANSI_COLOR_RESET "\n", id.c_str());
+    }
+}
+
+} // namespace
+
+TestRunOutput runExpressionTest(TestData& data, const std::string& rootPath, const std::string& id) {
+    TestRunOutput output(id);
+    const auto evaluateExpression = [&data](std::unique_ptr<style::expression::Expression>& expression,
+                                            TestResult& result) {
+        assert(expression);
+        std::vector<Value> outputs;
+        if (!data.inputs.empty()) {
+            for (const auto& input : data.inputs) {
+                auto evaluationResult = expression->evaluate(input.zoom, input.feature, input.heatmapDensity);
+                if (!evaluationResult) {
+                    std::unordered_map<std::string, Value> error{{"error", Value{evaluationResult.error().message}}};
+                    outputs.emplace_back(Value{std::move(error)});
+                } else {
+                    auto value = toValue(*evaluationResult);
+                    assert(value);
+                    outputs.emplace_back(Value{*value});
+                }
+            }
+        }
+        result.outputs = {Value{std::move(outputs)}};
+    };
+
+    // Parse expression
+    auto parsedExpression = parseExpression(data.document["expression"], data.spec, data.result);
+    output.expression = toJSON(data.result.expression.value_or(Value{}), 2, true);
+
+    // Evaluate expression
+    if (parsedExpression) {
+        evaluateExpression(parsedExpression, data.result);
+        output.serialized = toJSON(data.result.serialized.value_or(Value{}), 2, true);
+
+        // round trip
+        auto recompiledExpression = parseExpression(data.result.serialized, data.spec, data.recompiled);
+        if (recompiledExpression) {
+            evaluateExpression(recompiledExpression, data.recompiled);
+            rewriteRoundtrippedType(data.expected.compiled.serializedType,
+                                    data.recompiled.compiled.serializedType);
+        }
+    }
+
+    if (getenv("UPDATE")) {
+        output.passed = true;
+        updateTest(data);
+        writeTestData(data.document, rootPath, id);
+        return output;
+    }
+
+    bool compileOk = data.result.compiled == data.expected.compiled;
+    bool evalOk = compileOk && deepEqual(data.result.outputs, data.expected.outputs);
+
+    bool recompileOk = true;
+    bool roundTripOk = true;
+    bool serializationOk = true;
+
+    if (data.expected.compiled.success) {
+        serializationOk = compileOk && deepEqual(data.result.serialized, data.expected.serialized);
+        recompileOk = compileOk &&  data.recompiled.compiled == data.expected.compiled;
+        roundTripOk = recompileOk && deepEqual(data.recompiled.outputs, data.expected.outputs);
+    }
+
+    output.passed = compileOk && evalOk && recompileOk && roundTripOk && serializationOk;
+
+    if (!compileOk) {
+        auto resultValue = toValue(data.result.compiled);
+        auto expectedValue = toValue(data.expected.compiled);
+        output.text += "Compiled expression difference:\n"s +
+                        simpleDiff(resultValue, expectedValue) +
+                        "\n"s;
+    }
+
+    if (compileOk && !serializationOk) {
+        auto diff = simpleDiff(data.expected.serialized.value_or(Value{}),
+                               data.result.serialized.value_or(Value{}));
+        output.text += "Serialized expression difference:\n"s + diff + "\n"s;
+    }
+
+    if (compileOk && !recompileOk) {
+        auto recompiledValue = toValue(data.recompiled.compiled);
+        auto expectedValue = toValue(data.expected.compiled);
+        output.text += "Recompiled expression difference:\n"s +
+                        simpleDiff(recompiledValue, expectedValue) +
+                        "\n"s;
+    }
+
+    if (compileOk && !evalOk) {
+        auto diff = simpleDiff(data.expected.outputs.value_or(Value{}),
+                               data.result.outputs.value_or(Value{}));
+        output.text += "Expression outputs difference:\n"s + diff + "\n"s;
+    }
+
+    if (recompileOk && !roundTripOk) {
+        auto diff = simpleDiff(data.expected.outputs.value_or(Value{}),
+                               data.recompiled.outputs.value_or(Value{}));
+        output.text += "Roundtripped through serialize expression outputs difference:\n"s +
+                        diff + "\n"s;
+    }
+
+    return output;
+}

expression-test/expression_test_runner.hpp

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <vector>
+#include <string>
+
+class TestData;
+
+struct TestRunOutput {
+    TestRunOutput(std::string id_) : id(std::move(id_)) {}
+    std::string id;
+    bool passed = false;
+    std::string text;
+    std::string expression;
+    std::string serialized;
+};
+
+class TestStats {
+public:
+    std::size_t testCount() const {
+        return passed.size() + failed.size() + errored.size() +
+               ignorePassed.size() + ignoreFailed.size();
+    }
+
+    std::vector<TestRunOutput> passed;
+    std::vector<TestRunOutput> failed;
+    std::vector<TestRunOutput> errored;
+    std::vector<TestRunOutput> ignorePassed;
+    std::vector<TestRunOutput> ignoreFailed;
+    std::vector<std::string> ids;
+};
+
+TestRunOutput runExpressionTest(TestData&, const std::string& rootPath, const std::string& id);

expression-test/main.cpp

@@ -0,0 +1,75 @@
+#include "expression_test_parser.hpp"
+#include "expression_test_runner.hpp"
+#include "expression_test_logger.hpp"
+
+#include <random>
+
+int main(int argc, char** argv) {
+    // Parse args
+    std::vector<mbgl::filesystem::path> testPaths;
+    mbgl::filesystem::path rootPath;
+    bool shuffle;
+    uint32_t seed;
+    std::tie(rootPath, testPaths, shuffle, seed) = parseArguments(argc, argv);
+
+    // Parse ignores
+    const auto ignores = parseExpressionIgnores();
+
+    if (shuffle) {
+        printf(ANSI_COLOR_YELLOW "Shuffle seed: %d" ANSI_COLOR_RESET "\n", seed);
+        std::seed_seq sequence { seed };
+        std::mt19937 shuffler(sequence);
+        std::shuffle(testPaths.begin(), testPaths.end(), shuffler);
+    }
+
+    // Run tests, collect stats and output of each run.
+    TestStats stats;
+    for (const auto& path : testPaths) {
+        const auto& expectation = path.parent_path().string();
+        std::string id = expectation.substr(rootPath.string().length() + 1, expectation.length() - rootPath.string().length());
+        stats.ids.emplace_back(id);
+
+        bool shouldIgnore = false;
+        std::string ignoreReason;
+        const std::string ignoreName = "expression-tests/" + id;
+        const auto it = std::find_if(ignores.cbegin(), ignores.cend(), [&ignoreName] (const auto& ignore) { return ignore.id == ignoreName; });
+        if (it != ignores.end()) {
+            shouldIgnore = true;
+            ignoreReason = (*it).reason;
+        }
+
+        optional<TestRunOutput> testRun;
+        if (auto testData = parseTestData(path)) {
+            testRun = runExpressionTest(*testData, rootPath.string(), id);
+        }
+
+        if (!testRun) {
+            stats.errored.emplace_back(id);
+            printf(ANSI_COLOR_RED "* ERROR can't parse '%s' test" ANSI_COLOR_RESET "\n", id.c_str());
+            continue;
+        }
+
+        if (shouldIgnore) {
+            if (testRun->passed) {
+                stats.ignorePassed.emplace_back(std::move(*testRun));
+                printf(ANSI_COLOR_YELLOW "* PASSED ignored test %s (%s)" ANSI_COLOR_RESET "\n", id.c_str(), ignoreReason.c_str());
+            } else {
+                stats.ignoreFailed.emplace_back(std::move(*testRun));
+                printf(ANSI_COLOR_LIGHT_GRAY "* FAILED ignored test %s (%s)" ANSI_COLOR_RESET "\n", id.c_str(), ignoreReason.c_str());
+            }
+        } else {
+            if (testRun->passed) {
+                stats.passed.emplace_back(std::move(*testRun));
+                printf(ANSI_COLOR_GREEN "* PASSED %s" ANSI_COLOR_RESET "\n", id.c_str());
+            } else {
+                printf(ANSI_COLOR_RED "* FAILED %s\n%s" ANSI_COLOR_RESET, id.c_str(), testRun->text.c_str());
+                stats.failed.emplace_back(std::move(*testRun));
+            }
+        }
+    }
+
+    printStats(stats);
+    writeHTMLResults(stats, rootPath.string(), shuffle, seed);
+
+    return stats.errored.size() + stats.failed.size() == 0 ? 0 : 1;
+}