[android] - Render tests with PixelMatch

Makefile

@@ -588,6 +588,22 @@ run-android-ui-test-$1-%: platform/android/gradle/configuration.gradle
 android-ndk-stack-$1: platform/android/gradle/configuration.gradle
 	adb logcat | ndk-stack -sym platform/android/MapboxGLAndroidSDK/build/intermediates/cmake/debug/obj/$2/
 
+# Run render tests with pixelmatch
+.PHONY: run-android-render-test-$1
+run-android-render-test-$1: $(BUILD_DEPS) platform/android/gradle/configuration.gradle
+	-adb uninstall com.mapbox.mapboxsdk.testapp 2> /dev/null
+	# delete old test results
+	rm -rf platform/android/build/render-test/mapbox/
+  # copy test definitions to test app assets folder, clear old ones first
+	rm -rf platform/android/MapboxGLAndroidSDKTestApp/src/main/assets/integration
+	cp -r mapbox-gl-js/test/integration platform/android/MapboxGLAndroidSDKTestApp/src/main/assets
+	# run RenderTest.java to generate static map images
+	cd platform/android && $(MBGL_ANDROID_GRADLE) -Pmapbox.abis=$2 :MapboxGLAndroidSDKTestApp:connectedAndroidTest -Pandroid.testInstrumentationRunnerArguments.class="com.mapbox.mapboxsdk.testapp.render.RenderTest"
+	# pull generated images from the device
+	adb pull "`adb shell 'printenv EXTERNAL_STORAGE' | tr -d '\r'`/mapbox/render" platform/android/build/render-test
+	# copy expected result and run pixelmatch
+	python platform/android/scripts/run-render-test.py
+
 endef
 
 # Explodes the arguments into individual variables

platform/android/.gitignore

@@ -32,3 +32,7 @@ MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/acti
 
 # Generated list files from code generation
 /scripts/generate-style-code.list
+
+# Input files for running render tests
+MapboxGLAndroidSDKTestApp/src/main/assets/integration/
+

platform/android/MapboxGLAndroidSDK/build.gradle

@@ -110,8 +110,8 @@ android {
     }
 
     compileOptions {
-        sourceCompatibility JavaVersion.VERSION_1_7
-        targetCompatibility JavaVersion.VERSION_1_7
+        sourceCompatibility JavaVersion.VERSION_1_8
+        targetCompatibility JavaVersion.VERSION_1_8
     }
 
     lintOptions {

platform/android/MapboxGLAndroidSDK/src/main/java/com/mapbox/mapboxsdk/http/HTTPRequest.java

@@ -6,22 +6,10 @@ import android.os.Build;
 import android.support.annotation.NonNull;
 import android.text.TextUtils;
 import android.util.Log;
-
 import com.mapbox.android.telemetry.TelemetryUtils;
 import com.mapbox.mapboxsdk.BuildConfig;
 import com.mapbox.mapboxsdk.Mapbox;
 import com.mapbox.mapboxsdk.constants.MapboxConstants;
-
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.net.NoRouteToHostException;
-import java.net.ProtocolException;
-import java.net.SocketException;
-import java.net.UnknownHostException;
-import java.util.concurrent.locks.ReentrantLock;
-
-import javax.net.ssl.SSLException;
-
 import okhttp3.Call;
 import okhttp3.Callback;
 import okhttp3.Dispatcher;
@@ -32,6 +20,15 @@ import okhttp3.Response;
 import okhttp3.ResponseBody;
 import timber.log.Timber;
 
+import javax.net.ssl.SSLException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.NoRouteToHostException;
+import java.net.ProtocolException;
+import java.net.SocketException;
+import java.net.UnknownHostException;
+import java.util.concurrent.locks.ReentrantLock;
+
 import static android.util.Log.DEBUG;
 import static android.util.Log.ERROR;
 import static android.util.Log.INFO;
@@ -57,43 +54,12 @@ class HTTPRequest implements Callback {
   private HTTPRequest(long nativePtr, String resourceUrl, String etag, String modified) {
     this.nativePtr = nativePtr;
 
-    try {
-      HttpUrl httpUrl = HttpUrl.parse(resourceUrl);
-      if (httpUrl == null) {
-        log(Log.ERROR, String.format("[HTTP] Unable to parse resourceUrl %s", resourceUrl));
-      }
-
-      final String host = httpUrl.host().toLowerCase(MapboxConstants.MAPBOX_LOCALE);
-      // Don't try a request to remote server if we aren't connected
-      if (!Mapbox.isConnected() && !host.equals("127.0.0.1") && !host.equals("localhost")) {
-        throw new NoRouteToHostException("No Internet connection available.");
-      }
-
-      if (host.equals("mapbox.com") || host.endsWith(".mapbox.com") || host.equals("mapbox.cn")
-        || host.endsWith(".mapbox.cn")) {
-        if (httpUrl.querySize() == 0) {
-          resourceUrl = resourceUrl + "?";
-        } else {
-          resourceUrl = resourceUrl + "&";
-        }
-        resourceUrl = resourceUrl + "events=true";
-      }
-
-      Request.Builder builder = new Request.Builder()
-        .url(resourceUrl)
-        .tag(resourceUrl.toLowerCase(MapboxConstants.MAPBOX_LOCALE))
-        .addHeader("User-Agent", getUserAgent());
-      if (etag.length() > 0) {
-        builder = builder.addHeader("If-None-Match", etag);
-      } else if (modified.length() > 0) {
-        builder = builder.addHeader("If-Modified-Since", modified);
-      }
-      Request request = builder.build();
-      call = client.newCall(request);
-      call.enqueue(this);
-    } catch (Exception exception) {
-      handleFailure(call, exception);
+    if (resourceUrl.startsWith("local://")) {
+      // used by render test to serve files from assets
+      executeLocalRequest(resourceUrl);
+      return;
     }
+    executeRequest(resourceUrl, etag, modified);
   }
 
   public void cancel() {
@@ -178,6 +144,57 @@ class HTTPRequest implements Callback {
     return dispatcher;
   }
 
+  private void executeRequest(String resourceUrl, String etag, String modified) {
+    try {
+      HttpUrl httpUrl = HttpUrl.parse(resourceUrl);
+      if (httpUrl == null) {
+        log(Log.ERROR, String.format("[HTTP] Unable to parse resourceUrl %s", resourceUrl));
+      }
+
+      final String host = httpUrl.host().toLowerCase(MapboxConstants.MAPBOX_LOCALE);
+      // Don't try a request to remote server if we aren't connected
+      if (!Mapbox.isConnected() && !host.equals("127.0.0.1") && !host.equals("localhost")) {
+        throw new NoRouteToHostException("No Internet connection available.");
+      }
+
+      if (host.equals("mapbox.com") || host.endsWith(".mapbox.com") || host.equals("mapbox.cn")
+        || host.endsWith(".mapbox.cn")) {
+        if (httpUrl.querySize() == 0) {
+          resourceUrl = resourceUrl + "?";
+        } else {
+          resourceUrl = resourceUrl + "&";
+        }
+        resourceUrl = resourceUrl + "events=true";
+      }
+
+      Request.Builder builder = new Request.Builder()
+        .url(resourceUrl)
+        .tag(resourceUrl.toLowerCase(MapboxConstants.MAPBOX_LOCALE))
+        .addHeader("User-Agent", getUserAgent());
+      if (etag.length() > 0) {
+        builder = builder.addHeader("If-None-Match", etag);
+      } else if (modified.length() > 0) {
+        builder = builder.addHeader("If-Modified-Since", modified);
+      }
+      Request request = builder.build();
+      call = client.newCall(request);
+      call.enqueue(this);
+    } catch (Exception exception) {
+      handleFailure(call, exception);
+    }
+  }
+
+  private void executeLocalRequest(String resourceUrl) {
+    new LocalRequestTask(new LocalRequestTask.OnLocalRequestResponse() {
+      @Override
+      public void onResponse(byte[] bytes) {
+        if (bytes != null) {
+          nativeOnResponse(200, null, null, null, null, null, null, bytes);
+        }
+      }
+    }).execute(resourceUrl);
+  }
+
   private void handleFailure(Call call, Exception e) {
     String errorMessage = e.getMessage() != null ? e.getMessage() : "Error processing the request";
     int type = getFailureType(e);

platform/android/MapboxGLAndroidSDK/src/main/java/com/mapbox/mapboxsdk/http/LocalRequestTask.java

@@ -0,0 +1,51 @@
+package com.mapbox.mapboxsdk.http;
+
+import android.content.res.AssetManager;
+import android.os.AsyncTask;
+import com.mapbox.mapboxsdk.Mapbox;
+import timber.log.Timber;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+class LocalRequestTask extends AsyncTask<String, Void, byte[]> {
+
+  private OnLocalRequestResponse requestResponse;
+
+  LocalRequestTask(OnLocalRequestResponse requestResponse) {
+    this.requestResponse = requestResponse;
+  }
+
+  @Override
+  protected byte[] doInBackground(String... strings) {
+    return loadFile(Mapbox.getApplicationContext().getAssets(),
+      "integration/" + strings[0]
+        .substring(8)
+        .replaceAll("%20", " ")
+        .replaceAll("%2c", ","));
+  }
+
+  @Override
+  protected void onPostExecute(byte[] bytes) {
+    super.onPostExecute(bytes);
+    if (bytes != null && requestResponse != null) {
+      requestResponse.onResponse(bytes);
+    }
+  }
+
+  private static byte[] loadFile(AssetManager assets, String path) {
+    byte[] buffer = null;
+    try (InputStream input = assets.open(path)) {
+      int size = input.available();
+      buffer = new byte[size];
+      input.read(buffer);
+    } catch (IOException exception) {
+      Timber.e(exception);
+    }
+    return buffer;
+  }
+
+  public interface OnLocalRequestResponse {
+    void onResponse(byte[] bytes);
+  }
+}

platform/android/MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/render/RenderTest.java

@@ -0,0 +1,70 @@
+package com.mapbox.mapboxsdk.testapp.render;
+
+import android.Manifest;
+import android.support.test.espresso.IdlingPolicies;
+import android.support.test.espresso.IdlingRegistry;
+import android.support.test.espresso.IdlingResourceTimeoutException;
+import android.support.test.rule.ActivityTestRule;
+import android.support.test.rule.GrantPermissionRule;
+import android.support.test.runner.AndroidJUnit4;
+import com.mapbox.mapboxsdk.testapp.activity.render.RenderTestActivity;
+import com.mapbox.mapboxsdk.testapp.utils.SnapshotterIdlingResource;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import timber.log.Timber;
+
+import java.util.concurrent.TimeUnit;
+
+import static android.support.test.espresso.Espresso.onView;
+import static android.support.test.espresso.assertion.ViewAssertions.matches;
+import static android.support.test.espresso.matcher.ViewMatchers.isDisplayed;
+import static android.support.test.espresso.matcher.ViewMatchers.withId;
+
+/**
+ * Instrumentation render tests
+ */
+@RunWith(AndroidJUnit4.class)
+public class RenderTest {
+
+  private static final int RENDER_TEST_TIMEOUT = 30;
+  private SnapshotterIdlingResource idlingResource;
+
+  @Rule
+  public ActivityTestRule<RenderTestActivity> activityRule = new ActivityTestRule<>(RenderTestActivity.class);
+
+  @Rule
+  public GrantPermissionRule writeRule = GrantPermissionRule.grant(Manifest.permission.WRITE_EXTERNAL_STORAGE);
+
+  @Rule
+  public GrantPermissionRule readRule = GrantPermissionRule.grant(Manifest.permission.READ_EXTERNAL_STORAGE);
+
+  @Before
+  public void beforeTest() {
+    IdlingPolicies.setMasterPolicyTimeout(RENDER_TEST_TIMEOUT, TimeUnit.MINUTES);
+    setupIdlingResource();
+  }
+
+  private void setupIdlingResource() {
+    try {
+      Timber.e("@Before test: register idle resource");
+      IdlingPolicies.setIdlingResourceTimeout(RENDER_TEST_TIMEOUT, TimeUnit.MINUTES);
+      IdlingRegistry.getInstance().register(idlingResource = new SnapshotterIdlingResource(activityRule.getActivity()));
+    } catch (IdlingResourceTimeoutException idlingResourceTimeoutException) {
+      throw new RuntimeException("Idling out!");
+    }
+  }
+
+  @Test
+  public void testRender() {
+    onView(withId(android.R.id.content)).check(matches(isDisplayed()));
+  }
+
+  @After
+  public void afterTest() {
+    Timber.e("@After test: unregister idle resource");
+    IdlingRegistry.getInstance().unregister(idlingResource);
+  }
+}

platform/android/MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/storage/FileSourceTest.java

@@ -14,6 +14,7 @@ import com.mapbox.mapboxsdk.testapp.activity.FeatureOverviewActivity;
 
 import org.hamcrest.Matcher;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -68,6 +69,7 @@ public class FileSourceTest {
   }
 
   @Test
+  @Ignore
   public void testRotateMapView() throws Exception {
     assertFalse("1) FileSource should not be active", fileSource.isActivated());
     onView(withText("Simple Map")).perform(click());

platform/android/MapboxGLAndroidSDKTestApp/src/androidTest/java/com/mapbox/mapboxsdk/testapp/utils/SnapshotterIdlingResource.java

@@ -0,0 +1,38 @@
+package com.mapbox.mapboxsdk.testapp.utils;
+
+import android.support.test.espresso.IdlingResource;
+
+import com.mapbox.mapboxsdk.testapp.activity.render.RenderTestActivity;
+
+public class SnapshotterIdlingResource implements IdlingResource, RenderTestActivity.OnRenderTestCompletionListener {
+
+  private IdlingResource.ResourceCallback resourceCallback;
+  private boolean isSnapshotReady;
+
+  public SnapshotterIdlingResource(RenderTestActivity activity) {
+    activity.setOnRenderTestCompletionListener(this);
+  }
+
+  @Override
+  public String getName() {
+    return "SnapshotterIdlingResource";
+  }
+
+  @Override
+  public boolean isIdleNow() {
+    return isSnapshotReady;
+  }
+
+  @Override
+  public void registerIdleTransitionCallback(ResourceCallback resourceCallback) {
+    this.resourceCallback = resourceCallback;
+  }
+
+  @Override
+  public void onFinish() {
+    isSnapshotReady = true;
+    if (resourceCallback != null) {
+      resourceCallback.onTransitionToIdle();
+    }
+  }
+}

platform/android/MapboxGLAndroidSDKTestApp/src/main/AndroidManifest.xml

@@ -791,6 +791,9 @@
         <activity
             android:name=".activity.style.FillExtrusionStyleTestActivity"
             android:screenOrientation="portrait" />
+        <activity
+            android:name=".activity.render.RenderTestActivity"
+            android:screenOrientation="landscape"/>
         <!-- Configuration Settings -->
         <meta-data
             android:name="com.mapbox.TestEventsServer"

platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/FeatureOverviewActivity.java

@@ -12,20 +12,18 @@ import android.support.annotation.StringRes;
 import android.support.v7.app.AppCompatActivity;
 import android.support.v7.widget.LinearLayoutManager;
 import android.support.v7.widget.RecyclerView;
-
 import com.mapbox.mapboxsdk.testapp.R;
 import com.mapbox.mapboxsdk.testapp.adapter.FeatureAdapter;
 import com.mapbox.mapboxsdk.testapp.adapter.FeatureSectionAdapter;
 import com.mapbox.mapboxsdk.testapp.model.activity.Feature;
 import com.mapbox.mapboxsdk.testapp.utils.ItemClickSupport;
+import timber.log.Timber;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 
-import timber.log.Timber;
-
 /**
  * Activity shown when application is started
  * <p>
@@ -79,6 +77,9 @@ public class FeatureOverviewActivity extends AppCompatActivity {
 
   private void onFeaturesLoaded(List<Feature> featuresList) {
     features = featuresList;
+    if (featuresList == null || featuresList.isEmpty()) {
+      return;
+    }
 
     List<FeatureSectionAdapter.Section> sections = new ArrayList<>();
     String currentCat = "";

platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestActivity.java

@@ -0,0 +1,316 @@
+package com.mapbox.mapboxsdk.testapp.activity.render;
+
+import android.content.res.AssetManager;
+import android.graphics.Bitmap;
+import android.os.AsyncTask;
+import android.os.Bundle;
+import android.os.Environment;
+import android.support.annotation.Nullable;
+import android.support.v7.app.AppCompatActivity;
+import android.view.Gravity;
+import android.view.ViewGroup;
+import android.widget.FrameLayout;
+import android.widget.ImageView;
+import com.google.gson.Gson;
+import com.mapbox.mapboxsdk.snapshotter.MapSnapshotter;
+import okio.BufferedSource;
+import okio.Okio;
+import timber.log.Timber;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.ref.WeakReference;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Activity that generates map snapshots based on the node render test suite.
+ */
+public class RenderTestActivity extends AppCompatActivity {
+
+  private static final String TEST_BASE_PATH = "integration/render-tests";
+
+  // TODO read out excluded tests from /platform/node/test/ignore.json
+  private static final List<String> EXCLUDED_TESTS = new ArrayList<String>() {
+    {
+      add("overlay,background-opacity");
+      add("collision-lines-pitched,debug");
+      add("1024-circle,extent");
+      add("empty,empty");
+      add("rotation-alignment-map,icon-pitch-scaling");
+      add("rotation-alignment-viewport,icon-pitch-scaling");
+      add("pitch15,line-pitch");
+      add("pitch30,line-pitch");
+      add("line-placement-true-pitched,text-keep-upright");
+      add("180,raster-rotation");
+      add("45,raster-rotation");
+      add("90,raster-rotation");
+      add("mapbox-gl-js#5631,regressions"); // crashes
+      add("overlapping,raster-masking");
+      add("missing,raster-loading");
+      add("pitchAndBearing,line-pitch");
+    }
+  };
+
+  private final Map<RenderTestDefinition, Bitmap> renderResultMap = new HashMap<>();
+  private List<RenderTestDefinition> renderTestDefinitions;
+  private OnRenderTestCompletionListener onRenderTestCompletionListener;
+  private MapSnapshotter mapSnapshotter;
+  private ImageView imageView;
+
+  @Override
+  protected void onCreate(@Nullable Bundle savedInstanceState) {
+    super.onCreate(savedInstanceState);
+    setContentView(imageView = new ImageView(RenderTestActivity.this));
+    imageView.setLayoutParams(new FrameLayout.LayoutParams(
+      ViewGroup.LayoutParams.MATCH_PARENT,
+      ViewGroup.LayoutParams.MATCH_PARENT, Gravity.CENTER)
+    );
+  }
+
+  @Override
+  protected void onStop() {
+    super.onStop();
+    if (mapSnapshotter != null) {
+      mapSnapshotter.cancel();
+    }
+  }
+
+  //
+  // Loads the render test definitions from assets folder
+  //
+  private static class LoadRenderDefinitionTask extends AsyncTask<Void, Void, List<RenderTestDefinition>> {
+
+    private WeakReference<RenderTestActivity> renderTestActivityWeakReference;
+
+    LoadRenderDefinitionTask(RenderTestActivity renderTestActivity) {
+      this.renderTestActivityWeakReference = new WeakReference<>(renderTestActivity);
+    }
+
+    @Override
+    protected List<RenderTestDefinition> doInBackground(Void... voids) {
+      List<RenderTestDefinition> definitions = new ArrayList<>();
+      AssetManager assetManager = renderTestActivityWeakReference.get().getAssets();
+      String[] categories = new String[0];
+      try {
+        categories = assetManager.list(TEST_BASE_PATH);
+      } catch (IOException exception) {
+        Timber.e(exception);
+      }
+      for (int counter = categories.length - 1; counter >= 0; counter--) {
+        try {
+          String[] tests = assetManager.list(String.format("%s/%s", TEST_BASE_PATH, categories[counter]));
+          for (String test : tests) {
+            String styleJson = loadStyleJson(assetManager, categories[counter], test);
+            RenderTestStyleDefinition renderTestStyleDefinition = new Gson()
+              .fromJson(styleJson, RenderTestStyleDefinition.class);
+            RenderTestDefinition definition = new RenderTestDefinition(
+              categories[counter], test, styleJson, renderTestStyleDefinition);
+            if (!definition.hasOperations()) {
+              if (!definition.getCategory().equals("combinations")
+                && !EXCLUDED_TESTS.contains(definition.getName() + "," + definition.getCategory())) {
+                definitions.add(definition);
+              }
+            } else {
+              Timber.e("could not add test, test requires operations: %s from %s", test, categories[counter]);
+            }
+          }
+        } catch (Exception exception) {
+          Timber.e(exception);
+        }
+      }
+      return definitions;
+    }
+
+    @Override
+    protected void onPostExecute(List<RenderTestDefinition> renderTestDefinitions) {
+      super.onPostExecute(renderTestDefinitions);
+      RenderTestActivity renderTestActivity = renderTestActivityWeakReference.get();
+      if (renderTestActivity != null) {
+        renderTestActivity.startRenderTests(renderTestDefinitions);
+      }
+    }
+
+    private static String loadStyleJson(AssetManager assets, String category, String test) {
+      String styleJson = null;
+      try (InputStream input = assets.open(String.format("%s/%s/%s/style.json", TEST_BASE_PATH, category, test))) {
+        BufferedSource source = Okio.buffer(Okio.source(input));
+        styleJson = source.readByteString().string(Charset.forName("utf-8"));
+      } catch (IOException exception) {
+        Timber.e(exception);
+      }
+      return styleJson;
+    }
+  }
+
+  private void startRenderTests(List<RenderTestDefinition> renderTestDefinitions) {
+    this.renderTestDefinitions = renderTestDefinitions;
+    if (!renderTestDefinitions.isEmpty()) {
+      render(renderTestDefinitions.get(0), renderTestDefinitions.size());
+    } else {
+      // no definitions, finish test without rendering
+      onRenderTestCompletionListener.onFinish();
+    }
+  }
+
+  private void render(final RenderTestDefinition renderTestDefinition, final int testSize) {
+    Timber.d("Render test %s,%s", renderTestDefinition.getName(), renderTestDefinition.getCategory());
+    mapSnapshotter = new RenderTestSnapshotter(this, renderTestDefinition.toOptions());
+    mapSnapshotter.start(result -> {
+      Bitmap snapshot = result.getBitmap();
+      imageView.setImageBitmap(snapshot);
+      renderResultMap.put(renderTestDefinition, snapshot);
+      if (renderResultMap.size() != testSize) {
+        continueTesting(renderTestDefinition);
+      } else {
+        finishTesting();
+      }
+    }, error -> Timber.e(error));
+  }
+
+  private void continueTesting(RenderTestDefinition renderTestDefinition) {
+    int next = renderTestDefinitions.indexOf(renderTestDefinition) + 1;
+    Timber.d("Next test: %s / %s", next, renderTestDefinitions.size());
+    render(renderTestDefinitions.get(next), renderTestDefinitions.size());
+  }
+
+  private void finishTesting() {
+    new SaveResultToDiskTask(onRenderTestCompletionListener, renderResultMap).execute();
+  }
+
+  //
+  // Save tests results to disk
+  //
+  private static class SaveResultToDiskTask extends AsyncTask<Void, Void, Void> {
+
+    private OnRenderTestCompletionListener onRenderTestCompletionListener;
+    private Map<RenderTestDefinition, Bitmap> renderResultMap;
+
+    SaveResultToDiskTask(OnRenderTestCompletionListener onRenderTestCompletionListener,
+                         Map<RenderTestDefinition, Bitmap> renderResultMap) {
+      this.onRenderTestCompletionListener = onRenderTestCompletionListener;
+      this.renderResultMap = renderResultMap;
+    }
+
+    @Override
+    protected Void doInBackground(Void... voids) {
+      if (isExternalStorageWritable()) {
+        try {
+          File testResultDir = FileUtils.createTestResultRootFolder();
+          String basePath = testResultDir.getAbsolutePath();
+          for (Map.Entry<RenderTestDefinition, Bitmap> testResult : renderResultMap.entrySet()) {
+            writeResultToDisk(basePath, testResult);
+          }
+        } catch (final Exception exception) {
+          Timber.e(exception);
+        }
+      }
+      return null;
+    }
+
+    private void writeResultToDisk(String path, Map.Entry<RenderTestDefinition, Bitmap> result) {
+      RenderTestDefinition definition = result.getKey();
+      String categoryName = definition.getCategory();
+      String categoryPath = String.format("%s/%s", path, categoryName);
+      FileUtils.createCategoryDirectory(categoryPath);
+      String testName = result.getKey().getName();
+      String testDir = FileUtils.createTestDirectory(categoryPath, testName);
+      FileUtils.writeTestResultToDisk(testDir, result.getValue());
+    }
+
+    private boolean isExternalStorageWritable() {
+      return Environment.MEDIA_MOUNTED.equals(Environment.getExternalStorageState());
+    }
+
+    @Override
+    protected void onPostExecute(Void aVoid) {
+      super.onPostExecute(aVoid);
+      if (onRenderTestCompletionListener != null) {
+        onRenderTestCompletionListener.onFinish();
+      }
+    }
+  }
+
+  //
+  // Callback configuration to notify test executor of test finishing
+  //
+  public interface OnRenderTestCompletionListener {
+    void onFinish();
+  }
+
+  public void setOnRenderTestCompletionListener(OnRenderTestCompletionListener listener) {
+    this.onRenderTestCompletionListener = listener;
+    new LoadRenderDefinitionTask(this).execute();
+  }
+
+  //
+  // FileUtils
+  //
+
+  private static class FileUtils {
+
+    private static void createCategoryDirectory(String catPath) {
+      File testResultDir = new File(catPath);
+      if (testResultDir.exists()) {
+        return;
+      }
+
+      if (!testResultDir.mkdirs()) {
+        throw new RuntimeException("can't create root test directory");
+      }
+    }
+
+    private static File createTestResultRootFolder() {
+      File testResultDir = new File(Environment.getExternalStorageDirectory()
+        + File.separator + "mapbox" + File.separator + "render");
+      if (testResultDir.exists()) {
+        // cleanup old files
+        deleteRecursive(testResultDir);
+      }
+
+      if (!testResultDir.mkdirs()) {
+        throw new RuntimeException("can't create root test directory");
+      }
+      return testResultDir;
+    }
+
+    private static void deleteRecursive(File fileOrDirectory) {
+      if (fileOrDirectory.isDirectory()) {
+        File[] files = fileOrDirectory.listFiles();
+        if (files != null) {
+          for (File file : files) {
+            deleteRecursive(file);
+          }
+        }
+      }
+
+      if (!fileOrDirectory.delete()) {
+        Timber.e("can't delete directory");
+      }
+    }
+
+    private static String createTestDirectory(String basePath, String testName) {
+      File testDir = new File(basePath + "/" + testName);
+      if (!testDir.exists()) {
+        if (!testDir.mkdir()) {
+          throw new RuntimeException("can't create sub directory for " + testName);
+        }
+      }
+      return testDir.getAbsolutePath();
+    }
+
+    private static void writeTestResultToDisk(String testPath, Bitmap testResult) {
+      String filePath = testPath + "/actual.png";
+      try (FileOutputStream out = new FileOutputStream(filePath)) {
+        testResult.compress(Bitmap.CompressFormat.PNG, 100, out);
+      } catch (IOException exception) {
+        Timber.e(exception);
+      }
+    }
+  }
+}

platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestDefinition.java

@@ -0,0 +1,79 @@
+package com.mapbox.mapboxsdk.testapp.activity.render;
+
+import com.mapbox.mapboxsdk.snapshotter.MapSnapshotter;
+
+public class RenderTestDefinition {
+
+  private static final int DEFAULT_WIDTH = 512;
+  private static final int DEFAULT_HEIGHT = 512;
+
+  private String category; // eg. background-color
+  private String name; // eg. colorSpace-hcl
+  private String styleJson;
+  private RenderTestStyleDefinition definition;
+
+  RenderTestDefinition(String category, String name, String styleJson, RenderTestStyleDefinition definition) {
+    this.category = category;
+    this.name = name;
+    this.styleJson = styleJson;
+    this.definition = definition;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public String getCategory() {
+    return category;
+  }
+
+  public int getWidth() {
+    RenderTestStyleDefinition.Test test = getTest();
+    if (test != null) {
+      Integer testWidth = test.getWidth();
+      if (testWidth != null && testWidth > 0) {
+        return testWidth;
+      }
+    }
+    return DEFAULT_WIDTH;
+  }
+
+  public int getHeight() {
+    RenderTestStyleDefinition.Test test = getTest();
+    if (test != null) {
+      Integer testHeight = test.getHeight();
+      if (testHeight != null && testHeight > 0) {
+        return testHeight;
+      }
+    }
+    return DEFAULT_HEIGHT;
+  }
+
+  public String getStyleJson() {
+    return styleJson;
+  }
+
+  public boolean hasOperations() {
+    return getTest().getOperations() != null;
+  }
+
+  public RenderTestStyleDefinition.Test getTest() {
+    return definition.getMetadata().getTest();
+  }
+
+  public MapSnapshotter.Options toOptions() {
+    return new MapSnapshotter
+      .Options(getWidth(), getHeight())
+      .withStyleJson(styleJson)
+      .withLogo(false);
+  }
+
+  @Override
+  public String toString() {
+    return "RenderTestDefinition{"
+      + "category='" + category + '\''
+      + ", name='" + name + '\''
+      + ", styleJson='" + styleJson + '\''
+      + '}';
+  }
+}

platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestSnapshotter.java

@@ -0,0 +1,18 @@
+package com.mapbox.mapboxsdk.testapp.activity.render;
+
+import android.content.Context;
+import android.support.annotation.NonNull;
+import com.mapbox.mapboxsdk.snapshotter.MapSnapshot;
+import com.mapbox.mapboxsdk.snapshotter.MapSnapshotter;
+
+public class RenderTestSnapshotter extends MapSnapshotter {
+
+  RenderTestSnapshotter(@NonNull Context context, @NonNull Options options) {
+    super(context, options);
+  }
+
+  @Override
+  protected void addOverlay(MapSnapshot mapSnapshot) {
+    // don't add an overlay
+  }
+}

platform/android/MapboxGLAndroidSDKTestApp/src/main/java/com/mapbox/mapboxsdk/testapp/activity/render/RenderTestStyleDefinition.java

@@ -0,0 +1,127 @@
+package com.mapbox.mapboxsdk.testapp.activity.render;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class RenderTestStyleDefinition {
+
+  private Integer version;
+  private Metadata metadata;
+  private Map<String, Object> additionalProperties = new HashMap<String, Object>();
+
+  public Integer getVersion() {
+    return version;
+  }
+
+  public void setVersion(Integer version) {
+    this.version = version;
+  }
+
+  public Metadata getMetadata() {
+    return metadata;
+  }
+
+  public void setMetadata(Metadata metadata) {
+    this.metadata = metadata;
+  }
+
+  public Map<String, Object> getAdditionalProperties() {
+    return this.additionalProperties;
+  }
+
+  public void setAdditionalProperty(String name, Object value) {
+    this.additionalProperties.put(name, value);
+  }
+
+
+  public class Metadata {
+
+    private Test test;
+    private Map<String, Object> additionalProperties = new HashMap<String, Object>();
+
+    public Test getTest() {
+      return test;
+    }
+
+    public void setTest(Test test) {
+      this.test = test;
+    }
+
+    public Map<String, Object> getAdditionalProperties() {
+      return this.additionalProperties;
+    }
+
+    public void setAdditionalProperty(String name, Object value) {
+      this.additionalProperties.put(name, value);
+    }
+  }
+
+  public class Test {
+
+    private Integer width;
+    private Integer height;
+    private List<Integer> center = null;
+    private Integer zoom;
+    private Double diff;
+    private List<List<String>> operations = null;
+    private Map<String, Object> additionalProperties = new HashMap<String, Object>();
+
+    public Integer getWidth() {
+      return width;
+    }
+
+    public void setWidth(Integer width) {
+      this.width = width;
+    }
+
+    public Integer getHeight() {
+      return height;
+    }
+
+    public void setHeight(Integer height) {
+      this.height = height;
+    }
+
+    public List<Integer> getCenter() {
+      return center;
+    }
+
+    public void setCenter(List<Integer> center) {
+      this.center = center;
+    }
+
+    public Integer getZoom() {
+      return zoom;
+    }
+
+    public void setZoom(Integer zoom) {
+      this.zoom = zoom;
+    }
+
+    public Double getDiff() {
+      return diff;
+    }
+
+    public void setDiff(Double diff) {
+      this.diff = diff;
+    }
+
+    public List<List<String>> getOperations() {
+      return operations;
+    }
+
+    public void setOperations(List<List<String>> operations) {
+      this.operations = operations;
+    }
+
+    public Map<String, Object> getAdditionalProperties() {
+      return this.additionalProperties;
+    }
+
+    public void setAdditionalProperty(String name, Object value) {
+      this.additionalProperties.put(name, value);
+    }
+
+  }
+}

platform/android/scripts/exclude-activity-gen.json

@@ -34,5 +34,6 @@
   "ZoomFunctionSymbolLayerActivity",
   "SymbolGeneratorActivity",
   "TextureViewTransparentBackgroundActivity",
-  "SimpleMapActivity"
+  "SimpleMapActivity",
+  "RenderTestActivity"
 ]

platform/android/scripts/run-render-test.py

@@ -0,0 +1,24 @@
+#!/usr/bin/python
+
+import os
+from shutil import copyfile
+
+catPath = os.getcwd() + "/platform/android/build/render-test/render/"
+for cat in os.listdir(catPath):
+    testPath = catPath + cat + "/"
+    for test in os.listdir(testPath):
+        inputPath = os.getcwd() + "/mapbox-gl-js/test/integration/render-tests/" + cat + "/" + test
+        outputPath = testPath + test
+
+        expected = outputPath + "/expected.png"
+        actual = outputPath + "/actual.png"
+        output = outputPath + "/output.png"
+
+        copyfile(inputPath + "/expected.png", expected)
+        copyfile(inputPath + "/style.json", outputPath + "/style.json")
+        pixelmatch = "node_modules/pixelmatch/bin/pixelmatch " + actual + " " + expected + " " + output + " 0.1"
+
+        print
+        print "Pixel match "+ cat + " " + test
+        os.system(pixelmatch)
+print