From 83b32ec1db3dce493d79303bae9289b8e652f01b Mon Sep 17 00:00:00 2001 From: Mike Morris Date: Tue, 10 Feb 2015 17:07:49 -0800 Subject: [PATCH 1/4] move tests from mocha to tape --- Makefile | 2 +- package.json | 10 +-- test/file_source.test.js | 16 ++-- test/map.test.js | 185 ++++++++++++++++++++++----------------- 4 files changed, 119 insertions(+), 94 deletions(-) diff --git a/Makefile b/Makefile index b1756f017..4f83d2867 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ $(MBGL)/config/%.gypi: $(MBGL) $(MBGL)/configure .PHONY: test test: build - @`npm bin`/mocha -R spec + @`npm bin`/tape test/**/*.test.js .PHONY: clean clean: diff --git a/package.json b/package.json index ff2566bb3..0f53fe51e 100644 --- a/package.json +++ b/package.json @@ -3,12 +3,12 @@ "version": "0.0.1", "main": "lib/mbgl.node", "dependencies": { - "mkdirp": "^0.5.0", - "mocha": "^2.1.0", - "nan": "^1.4.1", - "request": "^2.51.0" + "nan": "^1.4.1" + }, + "devDependencies": { + "mkdirp": "^0.5.0", + "tape": "^3.5.0" }, - "devDependencies": {}, "scripts": { "install": "make", "test": "make test" diff --git a/test/file_source.test.js b/test/file_source.test.js index 1d7d86fe9..90a91de97 100644 --- a/test/file_source.test.js +++ b/test/file_source.test.js @@ -1,19 +1,21 @@ /* jshint node: true, unused: false */ -/* global describe, it */ 'use strict'; -var assert = require('assert'); + +var test = require('tape').test; var mbgl = require('..'); -describe('FileSource', function() { - - describe('constructor', function() { - it('must be called with new', function() { - assert.throws(function() { +test('FileSource', function(t) { + t.test('constructor', function(t) { + t.test('must be called with new', function(t) { + t.throws(function() { mbgl.FileSource(); }, /Use the new operator to create new FileSource objects/); + t.end(); }); + t.end(); }); + t.end(); }); diff --git a/test/map.test.js b/test/map.test.js index 1d542c7b3..c61263694 100644 --- a/test/map.test.js +++ b/test/map.test.js @@ -1,7 +1,7 @@ /* jshint node: true, unused: false */ -/* global describe, it, beforeEach, afterEach */ 'use strict'; -var assert = require('assert'); + +var test = require('tape').test; var mbgl = require('..'); var fs = require('fs'); @@ -12,153 +12,176 @@ mkdirp.sync('test/results'); var style = require('./fixtures/style.json'); -describe('Map', function() { +function setup(fileSource, callback) { + callback(new mbgl.Map(fileSource)); +} - describe('constructor', function() { - it('must be called with new', function() { - assert.throws(function() { +test('Map', function(t) { + t.test('constructor', function(t) { + t.test('must be called with new', function(t) { + t.throws(function() { mbgl.Map(); }, /Use the new operator to create new Map objects/); + + t.end(); }); - it('should require a FileSource object as first parameter', function() { - assert.throws(function() { + t.test('should require a FileSource object as first parameter', function(t) { + t.throws(function() { new mbgl.Map(); }, /Requires a FileSource as first argument/); - assert.throws(function() { + t.throws(function() { new mbgl.Map('fileSource'); }, /Requires a FileSource as first argument/); - assert.throws(function() { + t.throws(function() { new mbgl.Map({}); }, /Requires a FileSource as first argument/); + + t.end(); }); - it('should require the FileSource object to have request and cancel methods', function() { + t.test('should require the FileSource object to have request and cancel methods', function(t) { var fileSource = new mbgl.FileSource(); - assert.throws(function() { + t.throws(function() { new mbgl.Map(fileSource); }, /FileSource must have a request member function/); fileSource.request = 'test'; - assert.throws(function() { + t.throws(function() { new mbgl.Map(fileSource); }, /FileSource must have a request member function/); fileSource.request = function() {}; - assert.throws(function() { + t.throws(function() { new mbgl.Map(fileSource); }, /FileSource must have a cancel member function/); fileSource.cancel = 'test'; - assert.throws(function() { + t.throws(function() { new mbgl.Map(fileSource); }, /FileSource must have a cancel member function/); fileSource.cancel = function() {}; - assert.doesNotThrow(function() { + t.doesNotThrow(function() { new mbgl.Map(fileSource); }); + + t.end(); }); + t.end(); }); - describe('load styles', function() { - var map; - + t.test('load styles', function(t) { var fileSource = new mbgl.FileSource(); fileSource.request = function() {}; fileSource.cancel = function() {}; - beforeEach(function() { - map = new mbgl.Map(fileSource); + t.test('requires a string or object as the first parameter', function(t) { + setup(fileSource, function(map) { + t.throws(function() { + map.load(); + }, /Requires a map style as first argument/); + + t.throws(function() { + map.load('invalid'); + }, /Expect either an object or array at root/); + t.end(); + }); }); - afterEach(function() { - map = null; + t.test('accepts an empty stylesheet string', function(t) { + setup(fileSource, function(map) { + t.doesNotThrow(function() { + map.load('{}'); + }); + t.end(); + }); }); - it('requires a string or object as the first parameter', function() { - assert.throws(function() { - map.load(); - }, /Requires a map style as first argument/); - - assert.throws(function() { - map.load('invalid'); - }, /Expect either an object or array at root/); + t.test('accepts a JSON stylesheet', function(t) { + setup(fileSource, function(map) { + t.doesNotThrow(function() { + map.load(style); + }); + t.end(); + }); }); - it('accepts an empty stylesheet string', function() { - map.load('{}'); + t.test('accepts a stringified stylesheet', function(t) { + setup(fileSource, function(map) { + t.doesNotThrow(function() { + map.load(JSON.stringify(style)); + }); + t.end(); + }); }); - it('accepts a JSON stylesheet', function() { - map.load(style); - }); - - it('accepts a stringified stylesheet', function() { - map.load(JSON.stringify(style)); - }); + t.end(); }); - describe('render argument requirements', function() { - var map; - + t.test('render argument requirements', function(t) { var fileSource = new mbgl.FileSource(); fileSource.request = function(req) { fs.readFile(path.join('test', req.url), function(err, data) { req.respond(err, { data: data }); - assert.ifError(err); + t.error(err); }); }; fileSource.cancel = function() {}; - beforeEach(function() { - map = new mbgl.Map(fileSource); - }); + t.test('requires an object as the first parameter', function(t) { + setup(fileSource, function(map) { + t.throws(function() { + map.render(); + }, /First argument must be an options object/); - afterEach(function() { - map = null; - }); + t.throws(function() { + map.render('invalid'); + }, /First argument must be an options object/); - it('requires an object as the first parameter', function() { - assert.throws(function() { - map.render(); - }, /First argument must be an options object/); - - assert.throws(function() { - map.render('invalid'); - }, /First argument must be an options object/); - }); - - it('requires a callback as the second parameter', function() { - assert.throws(function() { - map.render({}); - }, /Second argument must be a callback function/); - - assert.throws(function() { - map.render({}, 'invalid'); - }, /Second argument must be a callback function/); - }); - - it('requires a style to be set', function(done) { - map.render({}, function(err) { - assert.ok(err); - assert.equal(err.message, 'Style is not set'); - done(); + t.end(); }); }); - it('returns an image', function(done) { - map.load(style); - map.render({}, function(err, data) { - assert.ifError(err); - fs.writeFileSync('test/results/image.png', data); - done(); + t.test('requires a callback as the second parameter', function(t) { + setup(fileSource, function(map) { + t.throws(function() { + map.render({}); + }, /Second argument must be a callback function/); + + t.throws(function() { + map.render({}, 'invalid'); + }, /Second argument must be a callback function/); + + t.end(); + }); + }); + + t.test('requires a style to be set', function(t) { + setup(fileSource, function(map) { + map.render({}, function(err) { + t.ok(err); + t.equal(err.message, 'Style is not set'); + t.end(); + }); + }); + }); + + t.test('returns an image', function(t) { + setup(fileSource, function(map) { + map.load(style); + map.render({}, function(err, data) { + t.error(err); + fs.writeFileSync('test/results/image.png', data); + t.end(); + }); }); }); }); + t.end(); }); From b87cff289c1b65757ab0d652f09a5a12470c9cc2 Mon Sep 17 00:00:00 2001 From: Mike Morris Date: Tue, 10 Feb 2015 19:18:32 -0800 Subject: [PATCH 2/4] add mapbox-gl-test-suite render tests --- Makefile | 11 +- package.json | 2 + scripts/travis_test.sh | 2 +- test/{ => js}/file_source.test.js | 6 +- test/{ => js}/map.test.js | 8 +- test/render.test.js | 161 ++++++++++++++++++++++++++++++ 6 files changed, 180 insertions(+), 10 deletions(-) rename test/{ => js}/file_source.test.js (78%) rename test/{ => js}/map.test.js (97%) create mode 100644 test/render.test.js diff --git a/Makefile b/Makefile index 4f83d2867..3df03e09d 100644 --- a/Makefile +++ b/Makefile @@ -49,9 +49,16 @@ build/Makefile: $(MBGL)/config/$(HOST).gypi $(MBGL)/config/%.gypi: $(MBGL) $(MBGL)/configure make -C $(MBGL) config/$*.gypi +.PHONY: test-suite +test-suite: build + @(`npm bin`/tape test/render.test.js || true) + +.PHONY: test-js +test-js: build + @`npm bin`/tape test/js/**/*.test.js + .PHONY: test -test: build - @`npm bin`/tape test/**/*.test.js +test: test-js test-suite .PHONY: clean clean: diff --git a/package.json b/package.json index 0f53fe51e..be6604529 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,8 @@ "nan": "^1.4.1" }, "devDependencies": { + "aws-sdk": "^2.1.9", + "mapbox-gl-test-suite": "git://github.com/mapbox/mapbox-gl-test-suite", "mkdirp": "^0.5.0", "tape": "^3.5.0" }, diff --git a/scripts/travis_test.sh b/scripts/travis_test.sh index 707293505..70fff88aa 100755 --- a/scripts/travis_test.sh +++ b/scripts/travis_test.sh @@ -6,5 +6,5 @@ set -o pipefail # Travis OS X has no GPU if [[ ${TRAVIS_OS_NAME} == "linux" ]]; then npm test - # ./scripts/deploy_results.sh + ./scripts/deploy_results.sh fi diff --git a/test/file_source.test.js b/test/js/file_source.test.js similarity index 78% rename from test/file_source.test.js rename to test/js/file_source.test.js index 90a91de97..72f2180f2 100644 --- a/test/file_source.test.js +++ b/test/js/file_source.test.js @@ -1,9 +1,9 @@ -/* jshint node: true, unused: false */ 'use strict'; -var test = require('tape').test; +/* jshint node: true */ -var mbgl = require('..'); +var test = require('tape'); +var mbgl = require('../..'); test('FileSource', function(t) { t.test('constructor', function(t) { diff --git a/test/map.test.js b/test/js/map.test.js similarity index 97% rename from test/map.test.js rename to test/js/map.test.js index c61263694..97daa8d52 100644 --- a/test/map.test.js +++ b/test/js/map.test.js @@ -1,16 +1,16 @@ -/* jshint node: true, unused: false */ 'use strict'; -var test = require('tape').test; +/* jshint node: true */ -var mbgl = require('..'); +var test = require('tape'); +var mbgl = require('../..'); var fs = require('fs'); var path = require('path'); var mkdirp = require('mkdirp'); mkdirp.sync('test/results'); -var style = require('./fixtures/style.json'); +var style = require('../fixtures/style.json'); function setup(fileSource, callback) { callback(new mbgl.Map(fileSource)); diff --git a/test/render.test.js b/test/render.test.js new file mode 100644 index 000000000..1329b7ef5 --- /dev/null +++ b/test/render.test.js @@ -0,0 +1,161 @@ +'use strict'; + +/* jshint node:true */ + +var test = require('tape'); +var mbgl = require('..'); +var fs = require('fs'); +var path = require('path'); +var mkdirp = require('mkdirp'); +var spawn = require('child_process').spawn; + +var suitePath = path.dirname(require.resolve('mapbox-gl-test-suite/package.json')); + +function template(name) { + return fs.readFileSync(require.resolve('mapbox-gl-test-suite/templates/' + name + '.html.tmpl')).toString(); +} + +var results = ''; +var resultTemplate = template('result'); + +function format(tmpl, kwargs) { + return tmpl.replace(/\{\{|\}\}|\{([^}]+)\}/g, function(match, key) { + if (match === '{{') return '{'; + if (match === '}}') return '}'; + return kwargs[key]; + }); +} + +function renderTest(style, info, base, key) { + var dir = path.join(suitePath, 'tests', base, key); + return function(t) { + var watchdog = setTimeout(function() { + t.fail('timed out after 20 seconds'); + }, 20000); + + t.once('end', function() { + clearTimeout(watchdog); + }); + + var fileSource = new mbgl.FileSource(); + fileSource.request = function(req) { + fs.readFile(path.join(suitePath, decodeURIComponent(req.url)), function(err, data) { + req.respond(err, { data: data }); + t.error(err); + }); + }; + fileSource.cancel = function() {}; + + var map = new mbgl.Map(fileSource); + map.load(style); + + map.render(info[key], function(err, image) { + t.error(err); + mkdirp.sync(dir); + + var expected = path.join(dir, 'expected.png'); + var actual = path.join(dir, 'actual.png'); + var diff = path.join(dir, 'diff.png'); + + if (process.env.UPDATE) { + fs.writeFile(expected, image, function(err) { + t.error(err); + t.end(); + }); + } else { + fs.writeFile(actual, image, function(err) { + t.error(err); + + var compare = spawn('compare', ['-metric', 'MAE', actual, expected, diff]); + var error = ''; + + compare.stderr.on('data', function (data) { + error += data.toString(); + }); + + compare.on('exit', function (code) { + // The compare program returns 2 on error otherwise 0 if the images are similar or 1 if they are dissimilar. + if (code === 2) { + writeResult(error.trim(), Infinity); + } else { + var match = error.match(/^\d+(?:\.\d+)?\s+\(([^\)]+)\)\s*$/); + var difference = match ? parseFloat(match[1]) : Infinity; + writeResult(match ? '' : error, difference); + } + }); + + compare.stdin.end(); + + function writeResult(error, difference) { + var allowedDifference = ('diff' in info) ? info.diff : 0.001; + var color = difference <= allowedDifference ? 'green' : 'red'; + + results += format(resultTemplate, { + name: base, + key: key, + color: color, + error: error ? '

' + error + '

' : '', + difference: difference, + zoom: info.zoom || 0, + center: info.center || [0, 0], + bearing: info.bearing || 0, + width: info.width || 512, + height: info.height || 512 + }); + + t.ok(difference <= allowedDifference); + t.end(); + } + }); + } + }); + }; +} + +function rewriteLocalSchema(uri) { + return uri.replace(/^local:\/\//, ''); +} + +var tests; + +if (process.argv[1] === __filename) { + tests = process.argv.slice(2); +} + +test('Render', function(t) { + fs.readdirSync(path.join(suitePath, 'tests')).forEach(function(dir) { + if (dir === 'index.html' || dir[0] === '.') return; + if (tests && tests.length && tests.indexOf(dir) < 0) return; + + var style = require(path.join(suitePath, 'tests', dir, 'style.json')), + info = require(path.join(suitePath, 'tests', dir, 'info.json')); + + for (var k in style.sources) { + var source = style.sources[k]; + + if (source.tiles) { + source.tiles = source.tiles.map(rewriteLocalSchema); + } + + if (source.url) { + source.url = rewriteLocalSchema(source.url); + } + } + + if (style.sprite) style.sprite = rewriteLocalSchema(style.sprite); + if (style.glyphs) style.glyphs = rewriteLocalSchema(style.glyphs); + + style = JSON.stringify(style); + + for (k in info) { + (info[k].native === false ? t.skip : t.test)(dir + ' ' + k, renderTest(style, info, dir, k)); + } + }); + + t.test('results', function(t) { + var p = path.join(suitePath, 'tests', 'index.html'); + fs.writeFileSync(p, format(template('results'), {results: results})); + console.warn('Results at: ' + p); + t.end(); + }); +}); From 2e7dc629565886a30304bebf374a8fd39f7076e2 Mon Sep 17 00:00:00 2001 From: Mike Morris Date: Wed, 11 Feb 2015 13:34:02 -0800 Subject: [PATCH 3/4] actual matches expected --- test/render.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/render.test.js b/test/render.test.js index 1329b7ef5..17f9bd228 100644 --- a/test/render.test.js +++ b/test/render.test.js @@ -103,7 +103,7 @@ function renderTest(style, info, base, key) { height: info.height || 512 }); - t.ok(difference <= allowedDifference); + t.ok(difference <= allowedDifference, 'actual matches expected'); t.end(); } }); From 2c5bd218de7b62a49fa44597d6dc24726472b930 Mon Sep 17 00:00:00 2001 From: Mike Morris Date: Wed, 11 Feb 2015 13:49:51 -0800 Subject: [PATCH 4/4] add concurrency test --- test/js/concurrency.test.js | 74 +++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 test/js/concurrency.test.js diff --git a/test/js/concurrency.test.js b/test/js/concurrency.test.js new file mode 100644 index 000000000..6a2ee32ca --- /dev/null +++ b/test/js/concurrency.test.js @@ -0,0 +1,74 @@ +'use strict'; + +/* jshint node:true */ + +var test = require('tape'); +var mbgl = require('../..'); +var fs = require('fs'); +var path = require('path'); + +var suitePath = path.dirname(require.resolve('mapbox-gl-test-suite/package.json')); + + +function renderTest(style, info, dir, key) { + return function (t) { + var completed = 0; + var remaining = 10; + var start = +new Date; + + var fileSource = new mbgl.FileSource(); + fileSource.request = function(req) { + fs.readFile(path.join(suitePath, decodeURIComponent(req.url)), function(err, data) { + req.respond(err, { data: data }); + t.error(err); + }); + }; + fileSource.cancel = function() {}; + + var map = new mbgl.Map(fileSource); + map.load(style); + + for (var i = 0; i < remaining; i++) { + map.render(info[key], function(err, image) { + t.error(err); + t.ok(true, 'render @ ' + ((+new Date) - start) + 'ms'); + if (++completed === remaining) t.end(); + }); + } + }; +} + +function rewriteLocalSchema(uri) { + return uri.replace(/^local:\/\//, ''); +} + +test('Concurrency', function(t) { + var dir = 'line-join'; + var k = 'round'; + + var style = require(path.join(suitePath, 'tests', dir, 'style.json')), + info = require(path.join(suitePath, 'tests', dir, 'info.json')); + + for (var k in style.sources) { + var source = style.sources[k]; + + if (source.tiles) { + source.tiles = source.tiles.map(rewriteLocalSchema); + } + + if (source.url) { + source.url = rewriteLocalSchema(source.url); + } + } + + if (style.sprite) style.sprite = rewriteLocalSchema(style.sprite); + if (style.glyphs) style.glyphs = rewriteLocalSchema(style.glyphs); + + style = JSON.stringify(style); + + for (k in info) { + t.test(dir + ' ' + k, renderTest(style, info, dir, k)); + } + + t.end(); +});