Use mapbox-gl-test-suite

.travis.yml

@@ -22,6 +22,7 @@ before_install:
 
 install:
 - make setup
+- npm install git+https://github.com/mapbox/mapbox-gl-test-suite.git
 
 before_script:
 - rm -rf mapnik-packaging/out/packages
@@ -30,10 +31,7 @@ script:
 - make linux -j4 BUILDTYPE=${BUILDTYPE}
 - make test -j4 BUILDTYPE=${BUILDTYPE}
 - ./scripts/run_tests.sh
-- ./scripts/compare_images.js
-
-after_script:
-- ./scripts/deploy_results.sh
+- (cd ./node_modules/mapbox-gl-test-suite/ && (./bin/compare_images.js; ./bin/deploy_results.sh))
 
 notifications:
   hipchat:

scripts/compare_images.js

@@ -1,117 +0,0 @@
-#!/usr/bin/env node
-
-var fs = require('fs');
-var path = require('path');
-var spawn = require('child_process').spawn;
-
-var base_dir = path.join(path.resolve('.'), 'test/fixtures/styles');
-
-var files = fs.readdirSync(base_dir).filter(function(name) {
-    return name.match(/\.info\.json$/);
-});
-
-var html =
-    '<style>\n' +
-    '    body { font-family: Helvetica; }\n' +
-    '    h2 a { color:white; text-decoration:none; }\n' +
-    '</style>\n' +
-    '<table>\n' +
-    '<tr>\n' +
-    '    <th>Actual</th>\n' +
-    '    <th>Expected</th>\n' +
-    '    <th>Info</th>\n' +
-    '</tr>\n';
-
-var exitCode = 0;
-var failures = 0;
-
-processFiles();
-
-function processFiles() {
-    if (!files.length) return done();
-
-    var name = files.shift();
-
-    var info = require(path.join(base_dir, name));
-
-    var keys = Object.keys(info);
-
-    processFileTest();
-
-    function processFileTest() {
-        if (!keys.length) return processFiles();
-
-        var key = keys.shift();
-
-        var base = path.basename(name, '.info.json');
-
-        var actual = path.join(base_dir, base + '/' + key + '.actual.png');
-        var expected = path.join(base_dir, base + '/' + key + '.expected.png');
-        var diff = path.join(base_dir, base + '/' + key + '.diff.png');
-
-        var compare = spawn('compare', ['-metric', 'MAE', actual, expected, diff ]);
-        var error = '';
-        compare.stderr.on('data', function(data) {
-            error += data.toString();
-        });
-        compare.on('exit', function(code, signal) {
-            // The compare program returns 2 on error otherwise 0 if the images are similar or 1 if they are dissimilar.
-            if (code == 2) {
-                writeResult(base, key, info[key], error.trim(), Infinity);
-                exitCode = 2;
-            } else {
-                var match = error.match(/^\d+(?:\.\d+)?\s+\(([^\)]+)\)\s*$/);
-                var difference = match ? parseFloat(match[1]) : Infinity;
-                writeResult(base, key, info[key], match ? '' : error, difference);
-
-            }
-            processFileTest();
-        });
-        compare.stdin.end();
-    }
-}
-
-function writeResult(base, key, info, error, difference) {
-    var color = 'green';
-    var allowedDifference = ('diff' in info) ? info.diff : 0.01;
-    if (difference > allowedDifference) {
-        color = 'red';
-        if (exitCode < 1) {
-            exitCode = 1;
-        }
-        failures++;
-    }
-
-    html +=
-        '<tr>\n' +
-        '    <td><img src="' + base + '/' + key + '.actual.png" onmouseover="this.src=\'' + base + '/' + key + '.expected.png\'" onmouseout="this.src=\'' + base + '/' + key + '.actual.png\'"></td>\n' +
-        '    <td><img src="' + base + '/' + key + '.expected.png" onmouseover="this.src=\'' + base + '/' + key + '.diff.png\'" onmouseout="this.src=\'' + base + '/' + key + '.expected.png\'"></td>\n' +
-        '    <td>\n' +
-        '        <h2 style="text-align:center; background:' + color + '"><a href="' + base + '.style.json">' + base + '/' + key + '</a></h2>\n' +
-        (error ? '        <p>' + error + '</p>\n' : '') +
-        '        <ul>\n' +
-        '            <li>diff: <strong>' + difference + '</strong></li>\n' +
-        '            <li>zoom: <strong>' + (info.zoom || 0) + '</strong></li>\n' +
-        '            <li>center: <strong>' + (info.center || [0, 0]) + '</strong></li>\n' +
-        '            <li>bearing: <strong>' + (info.bearing || 0) + '</strong></li>\n' +
-        '            <li>width: <strong>' + (info.width || 512) + '</strong></li>\n' +
-        '            <li>height: <strong>' + (info.height || 512) + '</strong></li>\n' +
-        '        </ul>\n' +
-        '    </td>\n' +
-        '</tr>\n'
-    ;
-}
-
-function done() {
-    html += "</table>\n";
-
-    fs.writeFileSync(path.join(base_dir, 'index.html'), html);
-    console.warn('Results at: ' + path.join(base_dir, 'index.html'));
-    if (failures) {
-        console.warn('\x1B[1m\x1B[31m' + failures + ' ' + (failures == 1 ? 'image doesn\'t' : 'images don\'t') + ' match\x1B[39m\x1B[22m');
-    } else {
-        console.warn('\x1B[1m\x1B[32mAll images match\x1B[39m\x1B[22m');
-    }
-
-    process.exit(exitCode);
-}

scripts/deploy_results.sh

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-set -o pipefail
-
-aws s3 cp test/fixtures/styles/ s3://mapbox-gl-testing/headless/$TRAVIS_JOB_NUMBER/ --acl public-read --recursive
-
-echo http://mapbox-gl-testing.s3.amazonaws.com/headless/$TRAVIS_JOB_NUMBER/index.html

test/fixtures/styles/icons.info.json

@@ -1,18 +0,0 @@
-{
-    "default": {
-        "zoom": 14,
-        "center": [52.499167, 13.418056],
-        "height": 256,
-        "log": [
-            [1, "INFO", "HttpRequest", 200, "sprites/outdoors.json"],
-            [1, "INFO", "HttpRequest", 200, "sprites/outdoors.png"],
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8803-5375.vector.pbf"],
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8802-5375.vector.pbf"],
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8803-5374.vector.pbf"],
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8802-5374.vector.pbf"],
-            [6, "INFO", "HttpRequest"],
-
-            [1, "INFO", "Sprite", "loaded sprites/outdoors"]
-        ]
-    }
-}

test/fixtures/styles/icons.style.json

@@ -1,33 +0,0 @@
-{
-  "version": 3,
-  "sources": {
-    "mapbox": {
-      "type": "vector",
-      "url": "tiles/{z}-{x}-{y}.vector.pbf",
-      "tileSize": 512,
-      "maxZoom": 14
-    }
-  },
-  "sprite": "sprites/outdoors",
-  "layers": [
-    {
-      "id": "background",
-      "type": "background",
-      "style": {
-        "background-color": "green"
-      }
-    },
-    {
-      "id": "poi",
-      "source": "mapbox",
-      "source-layer": "poi_label",
-      "type": "icon",
-      "filter": { "maki": "restaurant" },
-      "render": {
-        "icon-image": "{maki}-12",
-        "icon-size": 12,
-        "icon-allow-overlap": true
-      }
-    }
-  ]
-}

test/fixtures/styles/line-color.info.json

@@ -1,27 +0,0 @@
-{
-    "default": {
-        "zoom": 14,
-        "center": [52.499167, 13.418056],
-        "height": 256,
-        "log": [
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8803-5375.vector.pbf"],
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8802-5375.vector.pbf"],
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8803-5374.vector.pbf"],
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8802-5374.vector.pbf"],
-            [4, "INFO", "HttpRequest"]
-        ]
-    },
-    "colored": {
-        "zoom": 14,
-        "center": [52.499167, 13.418056],
-        "height": 256,
-        "classes": ["colored"],
-        "log": [
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8803-5375.vector.pbf"],
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8802-5375.vector.pbf"],
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8803-5374.vector.pbf"],
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8802-5374.vector.pbf"],
-            [4, "INFO", "HttpRequest"]
-        ]
-    }
-}

test/fixtures/styles/line-color.style.json

@@ -1,33 +0,0 @@
-{
-  "version": 3,
-  "sources": {
-    "mapbox": {
-      "type": "vector",
-      "url": "tiles/{z}-{x}-{y}.vector.pbf",
-      "tileSize": 512,
-      "maxZoom": 14
-    }
-  },
-  "layers": [
-    {
-      "id": "background",
-      "type": "background",
-      "style": {
-        "background-color": "white"
-      }
-    },
-    {
-      "id": "road",
-      "source": "mapbox",
-      "source-layer": "road",
-      "type": "line",
-      "style": {
-        "line-width": 2,
-        "line-color": "#3590db"
-      },
-      "style.colored": {
-        "line-color": "#222222"
-      }
-    }
-  ]
-}

test/fixtures/styles/road-width.info.json

@@ -1,45 +0,0 @@
-{
-    "z13.9": {
-        "zoom": 13.9,
-        "center": [52.499167, 13.418056],
-        "height": 256,
-        "log": [
-            [1, "INFO", "HttpRequest", 200, "tiles/13-4401-2687.vector.pbf"],
-            [1, "INFO", "HttpRequest"]
-        ]
-    },
-    "z14.0": {
-        "zoom": 14,
-        "center": [52.499167, 13.418056],
-        "height": 256,
-        "log": [
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8803-5375.vector.pbf"],
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8802-5375.vector.pbf"],
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8803-5374.vector.pbf"],
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8802-5374.vector.pbf"],
-            [4, "INFO", "HttpRequest"]
-        ]
-    },
-    "z14.1": {
-        "zoom": 14.1,
-        "center": [52.499167, 13.418056],
-        "height": 256,
-        "log": [
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8803-5375.vector.pbf"],
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8802-5375.vector.pbf"],
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8803-5374.vector.pbf"],
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8802-5374.vector.pbf"],
-            [4, "INFO", "HttpRequest"]
-        ]
-    },
-    "z14.2": {
-        "zoom": 14.2,
-        "center": [52.499167, 13.418056],
-        "height": 256,
-        "log": [
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8803-5374.vector.pbf"],
-            [1, "INFO", "HttpRequest", 200, "tiles/14-8802-5374.vector.pbf"],
-            [2, "INFO", "HttpRequest"]
-        ]
-    }
-}

test/fixtures/styles/road-width.style.json

@@ -1,31 +0,0 @@
-{
-  "version": 3,
-  "sources": {
-    "mapbox": {
-      "type": "vector",
-      "url": "tiles/{z}-{x}-{y}.vector.pbf",
-      "tileSize": 512,
-      "maxZoom": 14
-    }
-  },
-  "layers": [
-    {
-      "id": "background",
-      "type": "background",
-      "style": {
-        "background-color": "white"
-      }
-    },
-    {
-      "id": "road",
-      "source": "mapbox",
-      "source-layer": "road",
-      "type": "line",
-      "style": {
-        "line-width": {
-          "stops": [[13, 0], [13.999, 0], [14, 4], [14.1, 10], [14.2, 20]]
-        }
-      }
-    }
-  ]
-}

test/fixtures/styles/world-aa.info.json

@@ -1,9 +0,0 @@
-{
-    "plain": {
-        "center": [0, 0],
-        "zoom": 0,
-        "log": [
-            [ 1, "INFO", "HttpRequest", 200, "tiles/0-0-0.vector.pbf" ]
-        ]
-    }
-}

test/fixtures/styles/world-aa.style.json

@@ -1,29 +0,0 @@
-{
-  "version": 3,
-  "sources": {
-    "mapbox": {
-      "type": "vector",
-      "url": "tiles/{z}-{x}-{y}.vector.pbf",
-      "tileSize": 512,
-      "maxZoom": 14
-    }
-  },
-  "layers": [
-    {
-      "id": "background",
-      "type": "background",
-      "style": {
-        "background-color": "red"
-      }
-    },
-    {
-      "id": "water",
-      "source": "mapbox",
-      "source-layer": "water",
-      "type": "fill",
-      "style": {
-        "fill-color": "blue"
-      }
-    }
-  ]
-}

test/fixtures/styles/world-no-aa.info.json

@@ -1,10 +0,0 @@
-{
-    "plain": {
-        "center": [0, 0],
-        "zoom": 0,
-        "log": [
-            [1, "INFO", "HttpRequest", 200, "tiles/0-0-0.vector.pbf"],
-            [1, "INFO", "HttpRequest"]
-        ]
-    }
-}

test/fixtures/styles/world-no-aa.style.json

@@ -1,30 +0,0 @@
-{
-  "version": 3,
-  "sources": {
-    "mapbox": {
-      "type": "vector",
-      "url": "tiles/{z}-{x}-{y}.vector.pbf",
-      "tileSize": 512,
-      "maxZoom": 14
-    }
-  },
-  "layers": [
-    {
-      "id": "background",
-      "type": "background",
-      "style": {
-        "background-color": "red"
-      }
-    },
-    {
-      "id": "water",
-      "source": "mapbox",
-      "source-layer": "water",
-      "type": "fill",
-      "style": {
-        "fill-color": "blue",
-        "fill-antialias": false
-      }
-    }
-  ]
-}

test/headless.cpp

@@ -6,6 +6,8 @@
 #include <mbgl/util/std.hpp>
 
 #include <rapidjson/document.h>
+#include <rapidjson/writer.h>
+#include <rapidjson/stringbuffer.h>
 
 #include "../common/headless_view.hpp"
 
@@ -16,24 +18,47 @@
 const std::string base_directory = []{
     std::string fn = __FILE__;
     fn.erase(fn.find_last_of("/"));
-    return fn + "/fixtures/styles";
+    return fn + "/../node_modules/mapbox-gl-test-suite/";
 }();
 
 class HeadlessTest : public ::testing::TestWithParam<std::string> {};
 
+void ResolveLocalURL(rapidjson::Value& value, rapidjson::Document& doc) {
+    std::string str { value.GetString(), value.GetStringLength() };
+    str.replace(0, 8, base_directory); // local://
+    value.SetString(str.c_str(), str.length(), doc.GetAllocator());
+}
+
 TEST_P(HeadlessTest, render) {
     using namespace mbgl;
 
     const std::string &base = GetParam();
 
-    const std::string style = util::read_file(base_directory + "/" + base + ".style.json");
-    const std::string info = util::read_file(base_directory + "/" + base + ".info.json");
+    std::string style = util::read_file(base_directory + "tests/" + base + "/style.json");
+    std::string info = util::read_file(base_directory + "tests/" + base + "/info.json");
+
+    // Parse style.
+    rapidjson::Document styleDoc;
+    styleDoc.Parse<0>((const char *const)style.c_str());
+    ASSERT_EQ(false, styleDoc.HasParseError());
+    ASSERT_EQ(true, styleDoc.IsObject());
+
+    if (styleDoc.HasMember("sprite")) {
+        ResolveLocalURL(styleDoc["sprite"], styleDoc);
+    }
+
+    ResolveLocalURL(styleDoc["sources"]["mapbox"]["url"], styleDoc);
+
+    rapidjson::StringBuffer buffer;
+    rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
+    styleDoc.Accept(writer);
+    style = buffer.GetString();
 
     // Parse settings.
-    rapidjson::Document doc;
-    doc.Parse<0>((const char *const)info.c_str());
-    ASSERT_EQ(false, doc.HasParseError());
-    ASSERT_EQ(true, doc.IsObject());
+    rapidjson::Document infoDoc;
+    infoDoc.Parse<0>((const char *const)info.c_str());
+    ASSERT_EQ(false, infoDoc.HasParseError());
+    ASSERT_EQ(true, infoDoc.IsObject());
 
     Log::Set<FixtureLogBackend>();
 
@@ -41,15 +66,13 @@ TEST_P(HeadlessTest, render) {
     HeadlessView view;
     Map map(view);
 
-    for (auto it = doc.MemberBegin(), end = doc.MemberEnd(); it != end; it++) {
-        const FixtureLogBackend &log = Log::Set<FixtureLogBackend>();
-
+    for (auto it = infoDoc.MemberBegin(), end = infoDoc.MemberEnd(); it != end; it++) {
         const std::string name { it->name.GetString(), it->name.GetStringLength() };
         const rapidjson::Value &value = it->value;
         ASSERT_EQ(true, value.IsObject());
         if (value.HasMember("center")) ASSERT_EQ(true, value["center"].IsArray());
 
-        const std::string actual_image = base_directory + "/" + base + "/" + name +  ".actual.png";
+        const std::string actual_image = base_directory + "tests/" + base + "/" + name +  "/actual.png";
 
         const double zoom = value.HasMember("zoom") ? value["zoom"].GetDouble() : 0;
         const double bearing = value.HasMember("bearing") ? value["bearing"].GetDouble() : 0;
@@ -84,73 +107,22 @@ TEST_P(HeadlessTest, render) {
 
         const std::string image = util::compress_png(width, height, pixels.get(), true);
         util::write_file(actual_image, image);
-
-        if (value.HasMember("log")) {
-            const rapidjson::Value &js_log = value["log"];
-            ASSERT_EQ(true, js_log.IsArray());
-            for (rapidjson::SizeType i = 0; i < js_log.Size(); i++) {
-                const rapidjson::Value &js_entry = js_log[i];
-                ASSERT_EQ(true, js_entry.IsArray());
-                if (js_entry.Size() == 5) {
-                    const uint32_t count = js_entry[rapidjson::SizeType(0)].GetUint();
-                    const FixtureLogBackend::LogMessage message {
-                        EventSeverityClass(js_entry[rapidjson::SizeType(1)].GetString()),
-                        EventClass(js_entry[rapidjson::SizeType(2)].GetString()),
-                        js_entry[rapidjson::SizeType(3)].GetInt64(),
-                        js_entry[rapidjson::SizeType(4)].GetString()
-                    };
-                    ASSERT_EQ(count, log.count(message)) << "Message: " << message << "Full Log: " << std::endl << log.messages;
-                } else if (js_entry.Size() == 4) {
-                    const uint32_t count = js_entry[rapidjson::SizeType(0)].GetUint();
-                    if (js_entry[rapidjson::SizeType(3)].IsString()) {
-                        const FixtureLogBackend::LogMessage message {
-                            EventSeverityClass(js_entry[rapidjson::SizeType(1)].GetString()),
-                            EventClass(js_entry[rapidjson::SizeType(2)].GetString()),
-                            js_entry[rapidjson::SizeType(3)].GetString()
-                        };
-                        ASSERT_EQ(count, log.count(message)) << "Message: " << message << "Full Log: " << std::endl << log.messages;
-                    } else {
-                        const FixtureLogBackend::LogMessage message {
-                            EventSeverityClass(js_entry[rapidjson::SizeType(1)].GetString()),
-                            EventClass(js_entry[rapidjson::SizeType(2)].GetString()),
-                             js_entry[rapidjson::SizeType(3)].GetInt64()
-                        };
-                        ASSERT_EQ(count, log.count(message)) << "Message: " << message << "Full Log: " << std::endl << log.messages;
-                    }
-                } else if (js_entry.Size() == 3) {
-                    const uint32_t count = js_entry[rapidjson::SizeType(0)].GetUint();
-                    const FixtureLogBackend::LogMessage message {
-                        EventSeverityClass(js_entry[rapidjson::SizeType(1)].GetString()),
-                        EventClass(js_entry[rapidjson::SizeType(2)].GetString())
-                    };
-                    ASSERT_EQ(count, log.count(message)) << "Message: " << message << "Full Log: " << std::endl << log.messages;
-                } else {
-                    FAIL();
-                }
-            }
-        }
-
-        const auto &unchecked = log.unchecked();
-        if (unchecked.size()) {
-            std::cerr << "Unchecked Log Messages (" << base << "/" << name << "): " << std::endl << unchecked;
-        }
     }
 
 }
 
 INSTANTIATE_TEST_CASE_P(Headless, HeadlessTest, ::testing::ValuesIn([] {
     std::vector<std::string> names;
-    const std::string ending = ".info.json";
 
-    DIR *dir = opendir(base_directory.c_str());
+    DIR *dir = opendir((base_directory + "tests").c_str());
     if (dir == nullptr) {
         return names;
     }
 
     for (dirent *dp = nullptr; (dp = readdir(dir)) != nullptr;) {
         const std::string name = dp->d_name;
-        if (name.length() >= ending.length() && name.compare(name.length() - ending.length(), ending.length(), ending) == 0) {
-            names.push_back(name.substr(0, name.length() - ending.length()));
+        if (name != "." && name != ".." && name != "index.html") {
+            names.push_back(name);
         }
     }