diff --git a/.gitmodules b/.gitmodules index d84cfecbc..2aa6d52be 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,3 @@ -[submodule "test/suite"] - path = test/suite - url = https://github.com/mapbox/mapbox-gl-test-suite.git - [submodule ".mason"] path = .mason url = https://github.com/mapbox/mason.git diff --git a/package.json b/package.json index 3795ca8f5..0fd6887ba 100644 --- a/package.json +++ b/package.json @@ -26,16 +26,14 @@ ], "devDependencies": { "aws-sdk": "^2.1.47", - "mkdirp": "^0.5.1", - "pngjs": "^0.4.0", + "mapbox-gl-test-suite": "git+https://github.com/mapbox/mapbox-gl-test-suite.git#6cfd7eb20de77018a7f10770f56df9ce1f8a76ac", "request": "^2.61.0", - "st": "^0.5.5", "tape": "^4.2.0" }, "scripts": { "install": "node-pre-gyp install --fallback-to-build=false || make node", "test": "tape platform/node/test/js/**/*.test.js", - "test-suite": "tape platform/node/test/render.test.js" + "test-suite": "node platform/node/test/render.test.js" }, "gypfile": true, "binary": { diff --git a/platform/node/README.md b/platform/node/README.md index 870151fd2..72a08e853 100644 --- a/platform/node/README.md +++ b/platform/node/README.md @@ -92,6 +92,8 @@ The `kind` is an enum and defined in [`mbgl.Resource`](https://github.com/mapbox It has no significance for anything but serves as a hint to your implemention as to what sort of resource to expect. E.g., your implementation could choose caching strategies based on the expected file type. +THe `request` implementation should pass uncompressed data to `req.respond`. If you are downloading assets from a source that applies gzip transport encoding, the implementation must decompress the results before passing them on. + A sample implementation that reads files from disk would look like the following: ```js diff --git a/platform/node/test/compare.js b/platform/node/test/compare.js deleted file mode 100644 index 3e5221de5..000000000 --- a/platform/node/test/compare.js +++ /dev/null @@ -1,27 +0,0 @@ -var spawn = require('child_process').spawn; - -module.exports = function compare(actual, expected, diff, t, callback) { - var compare = spawn('compare', ['-metric', 'MAE', actual, expected, diff]); - var error = ''; - - compare.stderr.on('data', function(data) { - error += data.toString(); - }); - - compare.on('error', function(err) { - t.error(err); - }); - - compare.on('exit', function(code) { - // The compare program returns 2 on error otherwise 0 if the images are similar or 1 if they are dissimilar. - if (code === 2) { - callback(error.trim(), Infinity); - } else { - var match = error.match(/^\d+(?:\.\d+)?\s+\(([^\)]+)\)\s*$/); - var difference = match ? parseFloat(match[1]) : Infinity; - callback(match ? '' : error, difference); - } - }); - - compare.stdin.end(); -}; diff --git a/platform/node/test/expected/gzip/success.png b/platform/node/test/expected/gzip/success.png deleted file mode 100644 index de41e0fe2..000000000 Binary files a/platform/node/test/expected/gzip/success.png and /dev/null differ diff --git a/platform/node/test/expected/gzip/unhandled.png b/platform/node/test/expected/gzip/unhandled.png deleted file mode 100644 index ddb90d8b8..000000000 Binary files a/platform/node/test/expected/gzip/unhandled.png and /dev/null differ diff --git a/platform/node/test/expected/map/image.png b/platform/node/test/expected/map/image.png deleted file mode 100644 index de41e0fe2..000000000 Binary files a/platform/node/test/expected/map/image.png and /dev/null differ diff --git a/platform/node/test/js/consecutive.test.js b/platform/node/test/js/consecutive.test.js deleted file mode 100644 index 928ca1d83..000000000 --- a/platform/node/test/js/consecutive.test.js +++ /dev/null @@ -1,82 +0,0 @@ -'use strict'; - -/* jshint node:true */ - -var test = require('tape'); -var mbgl = require('../../../..'); -var fs = require('fs'); -var path = require('path'); - -var suitePath = path.join(__dirname, '../../../../test/suite'); - -function renderTest(style, info, dir, key) { - return function (t) { - var completed = 0; - var remaining = 10; - var start = +new Date; - - var options = {}; - options.request = function(req) { - fs.readFile(path.join(suitePath, decodeURIComponent(req.url)), function(err, data) { - req.respond(err, { data: data }); - t.error(err); - }); - }; - options.cancel = function() {}; - options.ratio = 1.0; - - var map = new mbgl.Map(options); - map.load(style); - - function render() { - map.render(info[key], function(err, image) { - t.error(err); - - t.ok(true, 'render @ ' + ((+new Date) - start) + 'ms'); - if (++completed === remaining) { - map.release(); - t.end(); - } else { - render(); - } - }); - } - - render(); - }; -} - -function rewriteLocalSchema(uri) { - return uri.replace(/^local:\/\//, ''); -} - -test('Consecutive', function(t) { - var dir = 'line-join'; - var k = 'round'; - - var style = require(path.join(suitePath, 'tests', dir, 'style.json')), - info = require(path.join(suitePath, 'tests', dir, 'info.json')); - - for (var k in style.sources) { - var source = style.sources[k]; - - if (source.tiles) { - source.tiles = source.tiles.map(rewriteLocalSchema); - } - - if (source.url) { - source.url = rewriteLocalSchema(source.url); - } - } - - if (style.sprite) style.sprite = rewriteLocalSchema(style.sprite); - if (style.glyphs) style.glyphs = rewriteLocalSchema(style.glyphs); - - style = JSON.stringify(style); - - for (k in info) { - t.test(dir + ' ' + k, renderTest(style, info, dir, k)); - } - - t.end(); -}); diff --git a/platform/node/test/js/gzip.test.js b/platform/node/test/js/gzip.test.js deleted file mode 100644 index 4545a18cf..000000000 --- a/platform/node/test/js/gzip.test.js +++ /dev/null @@ -1,125 +0,0 @@ -'use strict'; - -/* jshint node: true */ - -var test = require('tape').test; -var mbgl = require('../../../..'); -var fs = require('fs'); -var path = require('path'); -var mkdirp = require('mkdirp'); -var http = require('http'); -var request = require('request'); -var st = require('st'); -var style = require('../fixtures/style.json'); -var PNG = require('pngjs').PNG; -var compare = require('../compare.js'); - -var server = http.createServer(st({ path: path.join(__dirname, '..') })); -server.listen(0); - -function filePath(name) { - return ['expected', 'actual', 'diff'].reduce(function(prev, key) { - var dir = path.join(__dirname, '..', key, 'gzip'); - mkdirp.sync(dir); - prev[key] = path.join(dir, name); - return prev; - }, {}); -} - -function setup(options, callback) { - callback(new mbgl.Map(options)); -} - -function getOptions(gzip, t) { - return { - request: function(req) { - var parts = req.url.split('.'); - var filetype = parts[parts.length - 1]; - - request({ - url: 'http://localhost:' + server.address().port + path.join('/', req.url), - encoding: null, - gzip: filetype === 'pbf' ? gzip : true, - headers: { - 'Accept-Encoding': 'gzip' - } - }, function (err, res, body) { - t.error(err); - var response = {}; - response.data = res.body; - req.respond(null, response); - }); - }, - ratio: 1.0 - }; -} - -test('gzip', function(t) { - t.test('success', function(t) { - mbgl.on('message', function(msg) { - if (msg.severity == 'ERROR') t.error(msg); - }); - - setup(getOptions(true, t), function(map) { - map.load(style); - map.render({}, function(err, data) { - mbgl.removeAllListeners('message'); - map.release(); - - t.error(err); - - var filename = filePath('success.png'); - - var png = new PNG({ - width: data.width, - height: data.height - }); - - png.data = data.pixels; - - if (process.env.UPDATE) { - png.pack() - .pipe(fs.createWriteStream(filename.expected)) - .on('finish', t.end); - } else { - png.pack() - .pipe(fs.createWriteStream(filename.actual)) - .on('finish', function() { - compare(filename.actual, filename.expected, filename.diff, t, function(err, diff) { - t.error(err); - t.ok(diff <= 0.01, 'actual matches expected'); - t.end(); - }); - }); - } - }); - }); - }); - - t.test('unhandled', function(t) { - mbgl.once('message', function(msg) { - if (msg.severity == 'ERROR') { - t.ok(msg, 'emits error'); - t.equal(msg.class, 'Style'); - t.equal(msg.severity, 'ERROR'); - t.ok(msg.text.match(/pbf unknown field type exception/), 'error text matches'); - } - }); - - setup(getOptions(false, t), function(map) { - map.load(style); - map.render({}, function(err, data) { - map.release(); - - t.ok(err, 'returns error'); - t.ok(err.message.match(/Failed to parse/), 'error text matches'); - - t.end(); - }); - }); - }); - - t.test('teardown', function(t) { - server.close(t.end); - }); -}); diff --git a/platform/node/test/js/map.test.js b/platform/node/test/js/map.test.js index 09870abaa..c4e341632 100644 --- a/platform/node/test/js/map.test.js +++ b/platform/node/test/js/map.test.js @@ -1,24 +1,10 @@ 'use strict'; -/* jshint node: true */ - var test = require('tape'); var mbgl = require('../../../..'); var fs = require('fs'); var path = require('path'); -var mkdirp = require('mkdirp'); var style = require('../fixtures/style.json'); -var PNG = require('pngjs').PNG; -var compare = require('../compare.js'); - -function filePath(name) { - return ['expected', 'actual', 'diff'].reduce(function(prev, key) { - var dir = path.join(__dirname, '..', key, 'map'); - mkdirp.sync(dir); - prev[key] = path.join(dir, name); - return prev; - }, {}); -} test('Map', function(t) { t.test('must be constructed with new', function(t) { @@ -250,31 +236,53 @@ test('Map', function(t) { map.release(); - var filename = filePath('image.png'); - - var png = new PNG({ - width: data.width, - height: data.height - }); - - png.data = data.pixels; - - if (process.env.UPDATE) { - png.pack() - .pipe(fs.createWriteStream(filename.expected)) - .on('finish', t.end); - } else { - png.pack() - .pipe(fs.createWriteStream(filename.actual)) - .on('finish', function() { - compare(filename.actual, filename.expected, filename.diff, t, function(err, diff) { - t.error(err); - t.ok(diff <= 0.01, 'actual matches expected'); - t.end(); - }); - }); - } + t.ok(data.pixels); + t.equal(data.width, 512); + t.equal(data.height, 512); + t.end(); }); }); + + t.test('can be called several times in serial', function(t) { + var completed = 0; + var remaining = 10; + var start = +new Date; + + var map = new mbgl.Map(options); + map.load(style); + + function render() { + map.render({}, function(err, data) { + t.error(err); + + t.ok(true, 'render @ ' + ((+new Date) - start) + 'ms'); + if (++completed === remaining) { + map.release(); + t.end(); + } else { + render(); + } + }); + } + + render(); + }); + + t.skip('throws if called in parallel', function(t) { + var completed = 0; + var remaining = 10; + var start = +new Date; + + var map = new mbgl.Map(options); + map.load(style); + + t.throws(function() { + map.render({}, function() {}); + map.render({}, function() {}); + }); + + map.release(); + t.end(); + }); }); }); diff --git a/platform/node/test/render.test.js b/platform/node/test/render.test.js index 10a7afcb5..b9f13a28f 100644 --- a/platform/node/test/render.test.js +++ b/platform/node/test/render.test.js @@ -1,175 +1,28 @@ 'use strict'; -/* jshint node:true */ - -var test = require('tape'); var mbgl = require('../../..'); -var fs = require('fs'); -var path = require('path'); -var mkdirp = require('mkdirp'); -var PNG = require('pngjs').PNG; -var compare = require('./compare.js'); -var suitePath = path.join(__dirname, '../../../test/suite'); - -function template(name) { - return fs.readFileSync(path.join(suitePath, 'templates', name + '.html.tmpl')).toString(); -} - -var results = ''; -var resultTemplate = template('result'); - -function format(tmpl, kwargs) { - return tmpl.replace(/\{\{|\}\}|\{([^}]+)\}/g, function(match, key) { - if (match === '{{') return '{'; - if (match === '}}') return '}'; - return kwargs[key]; - }); -} - -function renderTest(style, info, base, key) { - var dir = path.join(suitePath, 'tests', base, key); - mkdirp.sync(dir); - - return function(t) { - var watchdog = setTimeout(function() { - t.fail('timed out after 20 seconds'); - }, 20000); - - t.once('end', function() { - clearTimeout(watchdog); - - if (map) { - map.release(); - map = null; - } - }); - - var options = {}; - options.request = function(req) { - var url = decodeURIComponent(req.url); - fs.readFile(path.join(suitePath, url), function(err, data) { - req.respond(err, { data: data }); - }); - }; - options.ratio = info[key].pixelRatio || 1; - - var map = new mbgl.Map(options); - map.load(style); - - map.render(info[key], function(err, data) { - if (err) { - t.error(err); - return t.end(); - } - - var expected = path.join(dir, 'expected.png'); - var actual = path.join(dir, 'actual.png'); - var diff = path.join(dir, 'diff.png'); - - var png = new PNG({ - width: data.width, - height: data.height - }); - - png.data = data.pixels; - - if (process.env.UPDATE) { - png.pack() - .pipe(fs.createWriteStream(expected)) - .on('finish', t.end); - } else { - png.pack() - .pipe(fs.createWriteStream(actual)) - .on('finish', function() { - compare(actual, expected, diff, t, function(err, diff) { - t.error(err); - - var allowed = 0.001; - - if ('diff' in info[key]) { - if (typeof info[key].diff === 'number') { - allowed = info[key].diff; - } else if ('native' in info[key].diff) { - allowed = info[key].diff.native; - } - } - - results += format(resultTemplate, { - name: base, - key: key, - color: diff <= allowed ? 'green' : 'red', - error: err ? '
' + err + '
' : '', - difference: diff, - zoom: info.zoom || 0, - center: info.center || [0, 0], - bearing: info.bearing || 0, - width: info.width || 512, - height: info.height || 512 - }); - - if (!info[key].ignored || !('native' in info[key].ignored)) { - t.ok(diff <= allowed, 'expected ' + diff + ' to be less than ' + allowed); - } - - t.end(); - }); - }); - } - }); - }; -} - -function rewriteLocalSchema(url) { - var regex = /^local:\/\//; - if (url instanceof Array) { - return url.map(function(str) { - return str.replace(regex, ''); - }); - } else if (typeof url === "string") { - return url.replace(regex, ''); - } -} +var suite = require('mapbox-gl-test-suite'); +var request = require('request'); var tests; -if (process.argv[1] === __filename) { +if (process.argv[1] === __filename && process.argv.length > 2) { tests = process.argv.slice(2); } -test('Render', function(t) { - fs.readdirSync(path.join(suitePath, 'tests')).forEach(function(dir) { - if (dir === 'index.html' || dir[0] === '.') return; - if (tests && tests.length && tests.indexOf(dir) < 0) return; - - var style = require(path.join(suitePath, 'tests', dir, 'style.json')), - info = require(path.join(suitePath, 'tests', dir, 'info.json')); - - for (var k in style.sources) { - var source = style.sources[k]; - - if (source.tiles) { - source.tiles = source.tiles.map(rewriteLocalSchema); - } - - if (source.url) { - source.url = rewriteLocalSchema(source.url); - } - } - - if (style.sprite) style.sprite = rewriteLocalSchema(style.sprite); - if (style.glyphs) style.glyphs = rewriteLocalSchema(style.glyphs); - - style = JSON.stringify(style); - - for (k in info) { - (info[k].native === false ? t.skip : t.test)(dir + ' ' + k, renderTest(style, info, dir, k)); +suite.run('native', {tests: tests}, function (style, options, callback) { + var map = new mbgl.Map({ + ratio: options.pixelRatio, + request: function (req) { + request(req.url, {encoding: null}, function (err, response, body) { + req.respond(err, {data: body}); + }); } }); - t.test('results', function(t) { - var p = path.join(suitePath, 'tests', 'index.html'); - fs.writeFileSync(p, format(template('results'), {results: results})); - console.warn('Results at: ' + p); - t.end(); + map.load(style); + map.render(options, function (err, result) { + map.release(); + callback(err, result && result.pixels); }); }); diff --git a/scripts/compare_images.sh b/scripts/compare_images.sh deleted file mode 100755 index 240772646..000000000 --- a/scripts/compare_images.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -o pipefail - -(cd ./test/suite/ && ./bin/compare_images.py native) diff --git a/scripts/linux/after_script.sh b/scripts/linux/after_script.sh deleted file mode 100755 index c38af0a79..000000000 --- a/scripts/linux/after_script.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -o pipefail - -if [ ! -z "${AWS_ACCESS_KEY_ID}" ] && [ ! -z "${AWS_SECRET_ACCESS_KEY}" ] ; then - # Install and add awscli to PATH for uploading the results - mapbox_time "install_awscli" \ - pip install --user awscli - export PATH="`python -m site --user-base`/bin:${PATH}" - - mapbox_time_start "deploy_results" - (cd ./test/suite/ && ./bin/deploy_results.sh) - mapbox_time_finish -fi diff --git a/scripts/linux/run.sh b/scripts/linux/run.sh index 42d2a16d9..31272ea1c 100755 --- a/scripts/linux/run.sh +++ b/scripts/linux/run.sh @@ -27,11 +27,5 @@ make test -j${JOBS} BUILDTYPE=${BUILDTYPE} # Test ################################################################################ -mapbox_time "checkout_test_suite" \ -git submodule update --init test/suite - mapbox_time "run_tests" \ make test-* BUILDTYPE=${BUILDTYPE} - -mapbox_time "compare_results" \ -./scripts/compare_images.sh diff --git a/scripts/main.mk b/scripts/main.mk index 65f283b21..3639a780a 100644 --- a/scripts/main.mk +++ b/scripts/main.mk @@ -46,10 +46,6 @@ SUBMODULES += src/mbgl/util/geojsonvt/geojsonvt.hpp src/mbgl/util/geojsonvt/geojsonvt.hpp: ./scripts/flock.py .git/Submodule.lock git submodule update --init src/mbgl/util/geojsonvt -SUBMODULES += test/suite/package.json -test/suite/package.json: - ./scripts/flock.py .git/Submodule.lock git submodule update --init test/suite - ifeq ($(HOST),ios) SUBMODULES += platform/ios/vendor/SMCalloutView/SMCalloutView.h platform/ios/vendor/SMCalloutView/SMCalloutView.h: diff --git a/scripts/node/after_script.sh b/scripts/node/after_script.sh index d78e820e9..237f9fe60 100755 --- a/scripts/node/after_script.sh +++ b/scripts/node/after_script.sh @@ -35,3 +35,16 @@ if test "${COMMIT_MESSAGE#*'[publish binary]'}" != "$COMMIT_MESSAGE"; then npm test fi fi + +if [ ! -z "${AWS_ACCESS_KEY_ID}" ] && [ ! -z "${AWS_SECRET_ACCESS_KEY}" ] ; then + # Install and add awscli to PATH for uploading the results + pip install --user awscli + export PATH="`python -m site --user-base`/bin:${PATH}" + + REPO_NAME=$(basename $TRAVIS_REPO_SLUG) + gzip --stdout node_modules/mapbox-gl-test-suite/tests/index.html | \ + aws s3 cp --acl public-read --content-encoding gzip --content-type text/html \ + - s3://mapbox/$REPO_NAME/tests/$TRAVIS_JOB_NUMBER/index.html + + echo http://mapbox.s3.amazonaws.com/$REPO_NAME/tests/$TRAVIS_JOB_NUMBER/index.html +fi diff --git a/scripts/node/run.sh b/scripts/node/run.sh index 8566fc3b3..22db3fded 100755 --- a/scripts/node/run.sh +++ b/scripts/node/run.sh @@ -26,24 +26,9 @@ npm install --build-from-source # Travis OS X has no GPU if [[ ${TRAVIS_OS_NAME} == "linux" ]]; then - mapbox_time "checkout_test_suite" \ - git submodule update --init test/suite - mapbox_time "run_tests" \ npm test mapbox_time "run_render_tests" \ npm run test-suite - - if [ ! -z "${AWS_ACCESS_KEY_ID}" ] && [ ! -z "${AWS_SECRET_ACCESS_KEY}" ] ; then - # Install and add awscli to PATH for uploading the results - mapbox_time "install_awscli" \ - pip install --user awscli - export PATH="`python -m site --user-base`/bin:${PATH}" - - pushd test/suite - mapbox_time "deploy_results" \ - ./bin/deploy_results.sh - popd - fi fi diff --git a/test/api/repeated_render.cpp b/test/api/repeated_render.cpp index f8b92b85b..3317b4e3a 100644 --- a/test/api/repeated_render.cpp +++ b/test/api/repeated_render.cpp @@ -25,7 +25,7 @@ TEST(API, RepeatedRender) { Map map(view, fileSource, MapMode::Still); { - map.setStyleJSON(style, "test/suite"); + map.setStyleJSON(style, ""); std::promise