diff --git a/.gitignore b/.gitignore
index cf1aea98a..80e156730 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 /build
 /lib
 /node_modules
-/test/results
+/test/results/*
+!/test/results/*-expected.png
 image.png
diff --git a/test/js/gz.test.js b/test/js/gz.test.js
index e6ca13a9b..228897edc 100644
--- a/test/js/gz.test.js
+++ b/test/js/gz.test.js
@@ -10,9 +10,11 @@ var mkdirp = require('mkdirp');
 var http = require('http');
 var request = require('request');
 var st = require('st');
+var spawn = require('child_process').spawn;
 var style = require('../../test/fixtures/style.json');
+var dir = 'test/results';
 
-mkdirp.sync('test/results');
+mkdirp.sync(dir);
 
 var server = http.createServer(st({path: __dirname + '/..'}));
 server.listen(2900);
@@ -61,10 +63,54 @@ test('unpacked gzip', function(t) {
     setup(getFileSource(true, t), function(map) {
         map.load(style);
         map.render({}, function(err, image) {
-            mbgl.compressPNG(image, function(err, data) {
-                fs.writeFileSync('test/results/image-gz-success.png', data);
-                mbgl.removeAllListeners('message');
-                t.end();
+            mbgl.compressPNG(image, function(err, image) {
+                t.error(err);
+
+                var expected = path.join(dir, 'gz-success-expected.png');
+                var actual = path.join(dir, 'gz-success-actual.png');
+                var diff = path.join(dir, 'gz-success-diff.png');
+
+                if (process.env.UPDATE) {
+                    fs.writeFile(expected, image, function(err) {
+                        t.error(err);
+                        mbgl.removeAllListeners('message');
+                        t.end();
+                    });
+                } else {
+                    fs.writeFile(actual, image, function(err) {
+
+                        var compare = spawn('compare', ['-metric', 'MAE', actual, expected, diff]);
+                        var error = '';
+
+                        compare.stderr.on('data', function(data) {
+                            error += data.toString();
+                        });
+
+                        compare.on('error', function(err) {
+                            t.error(err);
+                        });
+
+                        compare.on('exit', function (code) {
+                            // The compare program returns 2 on error otherwise 0 if the images are similar or 1 if they are dissimilar.
+                            if (code === 2) {
+                                writeResult(error.trim(), Infinity);
+                            } else {
+                                var match = error.match(/^\d+(?:\.\d+)?\s+\(([^\)]+)\)\s*$/);
+                                var difference = match ? parseFloat(match[1]) : Infinity;
+                                writeResult(match ? '' : error, difference);
+                            }
+                        });
+
+                        compare.stdin.end();
+
+                        function writeResult(error, difference) {
+                            t.ok(difference <= 0.001, 'actual matches expected');
+                            mbgl.removeAllListeners('message');
+                            t.end();
+                        }
+
+                    });
+                }
             });
         });
     });
@@ -72,22 +118,69 @@ test('unpacked gzip', function(t) {
 
 test('unhandled gzip', function(t) {
 
-    mbgl.once('message', function(msg) {
+    mbgl.on('message', function(msg) {
         t.ok(msg, 'emits error');
         t.equal(msg.class, 'ParseTile');
         t.equal(msg.severity, 'ERROR');
-        t.equal(msg.text, 'Parsing [0/0/0] failed: pbf unknown field type exception');
+        t.equal(msg.text.indexOf('failed: pbf unknown field type exception') !== -1, true);
     });
 
     setup(getFileSource(false, t), function(map) {
         map.load(style);
         map.render({}, function(err, image) {
-            mbgl.compressPNG(image, function(err, data) {
-                fs.writeFileSync('test/results/image-gz-fail.png', data);
-                t.ok(fs.statSync('test/results/image-gz-success.png')['size'] > fs.statSync('test/results/image-gz-fail.png')['size'], 'render is smaller than unpacked render');
-                server.close(t.end);
+            mbgl.compressPNG(image, function(err, image) {
+                t.error(err);
+
+                var expected = path.join(dir, 'gz-fail-expected.png');
+                var actual = path.join(dir, 'gz-fail-actual.png');
+                var diff = path.join(dir, 'gz-fail-diff.png');
+
+                if (process.env.UPDATE) {
+                    fs.writeFile(expected, image, function(err) {
+                        t.error(err);
+                        server.close(t.end);
+                    });
+                } else {
+                    fs.writeFile(actual, image, function(err) {
+
+                        var compare = spawn('compare', ['-metric', 'MAE', actual, expected, diff]);
+                        var error = '';
+
+                        compare.stderr.on('data', function(data) {
+                            error += data.toString();
+                        });
+
+                        compare.on('error', function(err) {
+                            t.error(err);
+                        });
+
+                        compare.on('exit', function (code) {
+                            // The compare program returns 2 on error otherwise 0 if the images are similar or 1 if they are dissimilar.
+                            if (code === 2) {
+                                writeResult(error.trim(), Infinity);
+                            } else {
+                                var match = error.match(/^\d+(?:\.\d+)?\s+\(([^\)]+)\)\s*$/);
+                                var difference = match ? parseFloat(match[1]) : Infinity;
+                                writeResult(match ? '' : error, difference);
+                            }
+                        });
+
+                        compare.stdin.end();
+
+                        function writeResult(error, difference) {
+                            t.ok(difference <= 0.001, 'actual matches expected');
+                            mbgl.removeAllListeners('message');
+                            server.close(t.end);
+                        }
+
+                    });
+                }
             });
         });
     });
 
 });
+
+
+
+
diff --git a/test/results/gz-fail-expected.png b/test/results/gz-fail-expected.png
new file mode 100644
index 000000000..ddb90d8b8
Binary files /dev/null and b/test/results/gz-fail-expected.png differ