[general] moving ios and android project under ./platform folder

render-test/CMakeLists.txt

@@ -0,0 +1,52 @@
+add_library(
+    mbgl-render-test STATIC EXCLUDE_FROM_ALL
+    ${PROJECT_SOURCE_DIR}/expression-test/test_runner_common.cpp
+    ${PROJECT_SOURCE_DIR}/expression-test/test_runner_common.hpp
+    ${PROJECT_SOURCE_DIR}/render-test/allocation_index.cpp
+    ${PROJECT_SOURCE_DIR}/render-test/allocation_index.hpp
+    ${PROJECT_SOURCE_DIR}/render-test/file_source.cpp
+    ${PROJECT_SOURCE_DIR}/render-test/file_source.hpp
+    ${PROJECT_SOURCE_DIR}/render-test/filesystem.hpp
+    ${PROJECT_SOURCE_DIR}/render-test/include/mbgl/render_test.hpp
+    ${PROJECT_SOURCE_DIR}/render-test/manifest_parser.cpp
+    ${PROJECT_SOURCE_DIR}/render-test/manifest_parser.hpp
+    ${PROJECT_SOURCE_DIR}/render-test/metadata.hpp
+    ${PROJECT_SOURCE_DIR}/render-test/parser.cpp
+    ${PROJECT_SOURCE_DIR}/render-test/parser.hpp
+    ${PROJECT_SOURCE_DIR}/render-test/render_test.cpp
+    ${PROJECT_SOURCE_DIR}/render-test/runner.cpp
+    ${PROJECT_SOURCE_DIR}/render-test/runner.hpp
+)
+
+if(CMAKE_SYSTEM_NAME STREQUAL iOS)
+    set_target_properties(mbgl-render-test PROPERTIES XCODE_ATTRIBUTE_IPHONEOS_DEPLOYMENT_TARGET "${IOS_DEPLOYMENT_TARGET}")
+    set_target_properties(mbgl-render-test PROPERTIES XCODE_ATTRIBUTE_ENABLE_BITCODE "YES")
+    set_target_properties(mbgl-render-test PROPERTIES XCODE_ATTRIBUTE_BITCODE_GENERATION_MODE bitcode)
+    set_target_properties(mbgl-render-test PROPERTIES XCODE_ATTRIBUTE_ONLY_ACTIVE_ARCH $<$<CONFIG:Debug>:YES>)
+endif()
+
+# FIXME: Should not use core private interface
+target_include_directories(
+    mbgl-render-test
+    PRIVATE ${PROJECT_SOURCE_DIR}/src ${PROJECT_SOURCE_DIR}/platform/default/include
+)
+
+target_include_directories(
+    mbgl-render-test
+    PUBLIC ${PROJECT_SOURCE_DIR}/render-test/include ${PROJECT_SOURCE_DIR}/include
+)
+
+include(${PROJECT_SOURCE_DIR}/vendor/boost.cmake)
+
+target_link_libraries(
+    mbgl-render-test
+    PRIVATE
+        Mapbox::Base::Extras::args
+        Mapbox::Base::Extras::filesystem
+        Mapbox::Base::pixelmatch-cpp
+        mbgl-compiler-options
+        mbgl-vendor-boost
+    PUBLIC mbgl-core
+)
+
+set_property(TARGET mbgl-render-test PROPERTY FOLDER Core)

render-test/allocation_index.cpp

@@ -0,0 +1,101 @@
+#include "allocation_index.hpp"
+
+#include <atomic>
+#include <cassert>
+#include <cstdlib>
+#include <mutex>
+#include <unordered_map>
+
+namespace {
+
+std::atomic_size_t indexedMemorySize{0};
+std::atomic_size_t indexedMemoryPeak{0};
+std::atomic_size_t allocationsCount{0};
+std::unordered_map<void*, size_t> memoryIndex;
+std::atomic_bool suppresIndexing{false};
+std::atomic_bool active{false};
+std::mutex indexMutex;
+
+class FlagGuard {
+public:
+    explicit FlagGuard(std::atomic_bool& flag_)
+        : flag(flag_) { flag = true; }
+    ~FlagGuard() { flag = false; }
+
+private:
+    std::atomic_bool& flag;
+};
+
+void addToIndex(std::size_t sz, void* ptr) {
+    std::lock_guard<std::mutex> mlk(indexMutex);
+    FlagGuard flk(suppresIndexing);
+    allocationsCount++;
+    indexedMemorySize += sz;
+    if (indexedMemoryPeak < indexedMemorySize) indexedMemoryPeak = size_t(indexedMemorySize);
+    memoryIndex[ptr] = sz;
+}
+
+void removeFromIndex(void* ptr) {
+    std::lock_guard<std::mutex> mlk(indexMutex);
+    FlagGuard flk(suppresIndexing);
+    auto it = memoryIndex.find(ptr);
+    if (it == memoryIndex.end()) return;
+
+    assert(indexedMemorySize >= it->second);
+    indexedMemorySize -= it->second;
+    memoryIndex.erase(it);
+}
+
+inline bool canModifyIndex() {
+    return active && !suppresIndexing;
+}
+
+} // namespace
+
+// static
+void AllocationIndex::setActive(bool active_) {
+    active = active_;
+}
+
+// static
+bool AllocationIndex::isActive() {
+    return active;
+}
+
+// static
+void AllocationIndex::reset() {
+    std::lock_guard<std::mutex> mlk(indexMutex);
+    FlagGuard flk(suppresIndexing);
+    memoryIndex.clear();
+    indexedMemorySize = 0;
+    allocationsCount = 0;
+    indexedMemoryPeak = 0;
+}
+
+// static
+void* AllocationIndex::allocate(size_t size) {
+    void *ptr = std::malloc(size);
+    if (ptr && canModifyIndex()) addToIndex(size, ptr);
+    return ptr;
+}
+
+// static
+void AllocationIndex::deallocate(void* ptr) noexcept {
+    if (canModifyIndex()) removeFromIndex(ptr);
+    std::free(ptr);
+}
+
+// static
+size_t AllocationIndex::getAllocatedSize() {
+    return indexedMemorySize;
+}
+
+// static
+size_t AllocationIndex::getAllocationsCount() {
+    return allocationsCount;
+}
+
+// static
+size_t AllocationIndex::getAllocatedSizePeak() {
+    return indexedMemoryPeak;
+}

render-test/allocation_index.hpp

@@ -0,0 +1,53 @@
+#pragma once
+
+#include <cstddef>
+
+class AllocationIndex {
+public:
+    AllocationIndex() = delete;
+    /**
+     * @brief Starts/stops allocated memory indexing.
+     */
+    static void setActive(bool);
+    /**
+     * @brief Returns memory indexing activity status.
+     */
+    static bool isActive();
+    /**
+     * @brief Resets the allocated memory index and all statistics data.
+     */
+    static void reset();
+    /**
+     * @brief Same as `malloc()` + indexes the allocated data,
+     * if indexing is active.
+     * 
+     * @return void* 
+     */
+    static void* allocate(size_t);
+    /**
+     * @brief Same as `free()` + removes the corresponding data from index,
+     * if these data are present in the index and indexing is active.
+     * 
+     * @return void* 
+     */
+    static void deallocate(void*) noexcept;
+    /**
+     * @brief Returns the size (in bytes) of allocated data, currently present in the index.
+     * 
+     * @return size_t
+     */
+    static size_t getAllocatedSize();
+    /**
+     * @brief Returns the total amount of allocations since indexing start.
+     * 
+     * @return size_t 
+     */
+    static size_t getAllocationsCount();
+
+    /**
+     * @brief Returns the maximum size (in bytes) of the allocated data in the index, since last `reset()` call.
+     * 
+     * @return size_t 
+     */
+    static size_t getAllocatedSizePeak();
+};

render-test/android/README.md

@@ -0,0 +1,3 @@
+# RenderTestRunner
+
+This app is a purely native application, with no Java source code, that can run **mbgl-render-test-runner** on android devices.

render-test/android/app/build.gradle

@@ -0,0 +1,53 @@
+apply plugin: 'com.android.application'
+
+android {
+    compileSdkVersion 28
+
+    defaultConfig {
+        applicationId = 'com.mapbox.mapboxsdk.maps.render_test_runner'
+        minSdkVersion 14
+        targetSdkVersion 28
+        def abi = 'all'
+        if (project.hasProperty('mapbox.abis')) {
+            abi = project.getProperty('mapbox.abis')
+        }
+        ndk {
+            if (abi != 'all') {
+                abiFilters abi.split(' ')
+            } else {
+                abiFilters 'armeabi-v7a', 'x86', 'arm64-v8a', 'x86_64'
+            }
+        }
+        externalNativeBuild {
+            cmake {
+                arguments '-DANDROID_CCACHE=ccache'
+                arguments '-DANDROID_STL=c++_static'
+                targets 'mbgl-render-test-runner'
+            }
+        }
+        android {
+            testBuildType 'release'
+        }
+
+        testInstrumentationRunner 'androidx.test.runner.AndroidJUnitRunner'
+    }
+    externalNativeBuild {
+        cmake {
+            version '3.10.2'
+            path '../../../CMakeLists.txt'
+        }
+    }
+    buildTypes {
+        release {
+            signingConfig signingConfigs.debug
+        }
+    }
+}
+
+dependencies {
+    implementation 'androidx.appcompat:appcompat:1.0.2'
+    implementation 'androidx.constraintlayout:constraintlayout:1.1.3'
+    androidTestImplementation 'androidx.test.ext:junit:1.1.0'
+    androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0-beta01'
+    androidTestImplementation 'androidx.test:rules:1.2.0-beta01'
+}

render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java

@@ -0,0 +1,32 @@
+package android.app;
+
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import androidx.test.filters.LargeTest;
+import androidx.test.rule.ActivityTestRule;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.Assert;
+import org.junit.runner.RunWith;
+import android.util.Log;
+
+@LargeTest
+@RunWith(AndroidJUnit4.class)
+public class NativeActivityTest {
+
+    @Rule
+    public ActivityTestRule<NativeActivity> mActivityTestRule = new ActivityTestRule<>(NativeActivity.class, false, false);
+
+    @Test(timeout = 1200000L)
+    public void runRenderTests() throws Exception {
+        Log.v("Test", "Start the test");
+        mActivityTestRule.launchActivity(null);
+        while (TestState.running) {
+            Log.v("Test", "Test is running");
+            Thread.sleep(1000L);
+        }
+        Log.v("Test", "All render tests are finished!");
+        Assert.assertTrue("All test cases are passed", TestState.testResult);
+
+    }
+}

render-test/android/app/src/androidTest/java/android/app/TestState.java

@@ -0,0 +1,6 @@
+package android.app;
+
+public class TestState {
+    static boolean running = true;
+    static boolean testResult = true;
+}

render-test/android/app/src/main/AndroidManifest.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+          package="com.example.native_activity"
+          android:versionCode="1"
+          android:versionName="1.0">
+  <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
+  <application
+      android:allowBackup="false"
+      android:fullBackupContent="false"
+      android:icon="@mipmap/ic_launcher"
+      android:label="@string/app_name"
+      android:hasCode="true">
+
+    <activity android:name="android.app.NativeActivity"
+              android:label="@string/app_name"
+              android:screenOrientation="portrait">
+      <meta-data android:name="android.app.lib_name"
+                 android:value="mbgl-render-test-runner" />
+      <intent-filter>
+        <action android:name="android.intent.action.MAIN" />
+        <category android:name="android.intent.category.LAUNCHER" />
+      </intent-filter>
+    </activity>
+  </application>
+
+</manifest>

render-test/android/app/src/main/assets/to_zip.txt

@@ -0,0 +1,10 @@
+mapbox-gl-js/test/integration/render-tests/
+mapbox-gl-js/test/integration/query-tests/
+metrics/expectations/
+metrics/ignores/
+metrics/tests/
+metrics/android-render-test-runner/
+metrics/android-render-test-runner-metrics.json
+metrics/android-render-test-runner-style.json
+metrics/cache-metrics.db
+metrics/cache-style.db

render-test/android/app/src/main/res/values/strings.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <string name="app_name">RenderTestRunner</string>
+</resources>

render-test/android/build.gradle

@@ -0,0 +1,16 @@
+buildscript {
+    repositories {
+       google()
+       jcenter()
+    }
+    dependencies {
+        classpath 'com.android.tools.build:gradle:3.5.1'
+    }
+}
+
+allprojects {
+    repositories {
+        google()
+        jcenter()
+    }
+}

render-test/android/gradle.properties

@@ -0,0 +1 @@
+org.gradle.jvmargs=-Xmx1536m

render-test/android/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,6 @@
+#Mon Oct 21 12:45:47 EEST 2019
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.4.1-all.zip

render-test/android/gradlew

@@ -0,0 +1,164 @@
+#!/usr/bin/env bash
+
+##############################################################################
+##
+##  Gradle start up script for UN*X
+##
+##############################################################################
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn ( ) {
+    echo "$*"
+}
+
+die ( ) {
+    echo
+    echo "$*"
+    echo
+    exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+case "`uname`" in
+  CYGWIN* )
+    cygwin=true
+    ;;
+  Darwin* )
+    darwin=true
+    ;;
+  MINGW* )
+    msys=true
+    ;;
+esac
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched.
+if $cygwin ; then
+    [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
+fi
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >&-
+APP_HOME="`pwd -P`"
+cd "$SAVED" >&-
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+        JAVACMD="$JAVA_HOME/bin/java"
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD="java"
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
+    MAX_FD_LIMIT=`ulimit -H -n`
+    if [ $? -eq 0 ] ; then
+        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+            MAX_FD="$MAX_FD_LIMIT"
+        fi
+        ulimit -n $MAX_FD
+        if [ $? -ne 0 ] ; then
+            warn "Could not set maximum file descriptor limit: $MAX_FD"
+        fi
+    else
+        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+    fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin ; then
+    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+
+    # We build the pattern for arguments to be converted via cygpath
+    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+    SEP=""
+    for dir in $ROOTDIRSRAW ; do
+        ROOTDIRS="$ROOTDIRS$SEP$dir"
+        SEP="|"
+    done
+    OURCYGPATTERN="(^($ROOTDIRS))"
+    # Add a user-defined pattern to the cygpath arguments
+    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+    fi
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    i=0
+    for arg in "$@" ; do
+        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
+
+        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
+            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+        else
+            eval `echo args$i`="\"$arg\""
+        fi
+        i=$((i+1))
+    done
+    case $i in
+        (0) set -- ;;
+        (1) set -- "$args0" ;;
+        (2) set -- "$args0" "$args1" ;;
+        (3) set -- "$args0" "$args1" "$args2" ;;
+        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+    esac
+fi
+
+# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
+function splitJvmOpts() {
+    JVM_OPTS=("$@")
+}
+eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
+JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+
+exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"

render-test/android/gradlew.bat

@@ -0,0 +1,90 @@
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windowz variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+if "%@eval[2+2]" == "4" goto 4NT_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+goto execute
+
+:4NT_args
+@rem Get arguments from the 4NT Shell from JP Software
+set CMD_LINE_ARGS=%$
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

render-test/android/settings.gradle

@@ -0,0 +1,2 @@
+include ':app'
+

render-test/file_source.cpp

@@ -0,0 +1,105 @@
+#include <mbgl/storage/file_source_manager.hpp>
+#include <mbgl/storage/resource.hpp>
+#include <mbgl/storage/resource_options.hpp>
+#include <mbgl/util/async_request.hpp>
+#include <mbgl/util/logging.hpp>
+
+#include <atomic>
+
+#include "file_source.hpp"
+
+namespace mbgl {
+
+std::atomic_size_t requestCount{0};
+std::atomic_size_t transferredSize{0};
+std::atomic_bool active{false};
+std::atomic_bool offline{true};
+
+ProxyFileSource::ProxyFileSource(std::shared_ptr<FileSource> defaultResourceLoader_, const ResourceOptions& options)
+    : defaultResourceLoader(std::move(defaultResourceLoader_)) {
+    assert(defaultResourceLoader);
+    if (offline) {
+        std::shared_ptr<FileSource> dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, options);
+        dbfs->setProperty(READ_ONLY_MODE_KEY, true);
+    }
+}
+
+ProxyFileSource::~ProxyFileSource() = default;
+
+std::unique_ptr<AsyncRequest> ProxyFileSource::request(const Resource& resource, Callback callback) {
+    auto transformed = resource;
+
+    // If offline, force always loading the resource from the cache
+    // so we don't make any network request.
+    if (offline) {
+        transformed.loadingMethod = Resource::LoadingMethod::CacheOnly;
+    }
+
+    // This is needed for compatibility with the style tests that
+    // are using local:// instead of http:// which is the schema
+    // we support for cached files.
+    if (transformed.url.compare(0, 8, "local://") == 0) {
+        transformed.url.replace(0, 8, "http://");
+
+        if (transformed.kind == Resource::Kind::Tile && transformed.tileData) {
+            transformed.tileData->urlTemplate.replace(0, 8, "http://");
+        }
+    }
+
+    if (transformed.url.compare(0, 22, "http://localhost:2900/") == 0) {
+        transformed.url.replace(0, 22, "http://");
+
+        if (transformed.kind == Resource::Kind::Tile && transformed.tileData) {
+            transformed.tileData->urlTemplate.replace(0, 22, "http://");
+        }
+    }
+
+    return defaultResourceLoader->request(transformed, [=](Response response) {
+        if (transformed.loadingMethod == Resource::LoadingMethod::CacheOnly && response.noContent) {
+            if (transformed.kind == Resource::Kind::Tile && transformed.tileData) {
+                mbgl::Log::Info(mbgl::Event::Database,
+                                "Resource not found in cache: %s (%s)",
+                                transformed.url.c_str(),
+                                transformed.tileData->urlTemplate.c_str());
+            } else {
+                mbgl::Log::Info(mbgl::Event::Database, "Resource not found in cache: %s", transformed.url.c_str());
+            }
+        }
+
+        std::size_t size = response.data != nullptr ? response.data->size() : 0;
+        if (active) {
+            requestCount++;
+            transferredSize += size;
+        }
+        callback(response);
+    });
+}
+
+// static
+void ProxyFileSource::setOffline(bool status) {
+    offline = status;
+}
+
+// static
+void ProxyFileSource::setTrackingActive(bool active_) {
+    active = active_;
+    requestCount = 0;
+    transferredSize = 0;
+}
+
+// static
+bool ProxyFileSource::isTrackingActive() {
+    return active;
+}
+
+// static
+size_t ProxyFileSource::getRequestCount() {
+    return requestCount;
+}
+
+// static
+size_t ProxyFileSource::getTransferredSize() {
+    return transferredSize;
+}
+
+} // namespace mbgl

render-test/file_source.hpp

@@ -0,0 +1,50 @@
+#pragma once
+
+#include <mbgl/storage/file_source.hpp>
+
+namespace mbgl {
+
+class ResourceOptions;
+
+class ProxyFileSource : public FileSource {
+public:
+    ProxyFileSource(std::shared_ptr<FileSource>, const ResourceOptions&);
+    ~ProxyFileSource();
+
+    std::unique_ptr<AsyncRequest> request(const Resource&, Callback) override;
+    bool canRequest(const Resource&) const override { return true; }
+
+    /**
+     * @brief Flag to change the networking mode of the file source.
+     */
+    static void setOffline(bool);
+
+    /**
+     * @brief Starts/stops metrics tracking.
+     */
+    static void setTrackingActive(bool);
+
+    /**
+     * @brief Returns metrics tracking status.
+     */
+    static bool isTrackingActive();
+
+    /**
+     * @brief Returns the total amount of requests.
+     *
+     * @return size_t
+     */
+    static size_t getRequestCount();
+
+    /**
+     * @brief Returns the size of transferred data (in bytes).
+     *
+     * @return size_t
+     */
+    static size_t getTransferredSize();
+
+private:
+    std::shared_ptr<FileSource> defaultResourceLoader;
+};
+
+} // namespace mbgl

render-test/filesystem.hpp

@@ -0,0 +1,9 @@
+#pragma once
+
+#include <ghc/filesystem.hpp>
+
+namespace mbgl {
+
+namespace filesystem = ghc::filesystem;
+
+} // namespace mbgl

render-test/include/mbgl/render_test.hpp

@@ -0,0 +1,9 @@
+#pragma once
+
+#include <functional>
+
+namespace mbgl {
+
+int runRenderTests(int argc, char* argv[], std::function<void()>);
+
+} // namespace mbgl

render-test/ios/Info.plist

@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN""http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>CFBundleDevelopmentRegion</key>
+  <string>en</string>
+  <key>CFBundleDisplayName</key>
+  <string>${PRODUCT_NAME}</string>
+  <key>CFBundleExecutable</key>
+  <string>RenderTestApp</string>
+  <key>CFBundleIdentifier</key>
+  <string>com.mapbox.RenderTestApp</string>
+  <key>CFBundleGetInfoString</key>
+  <string>com.mapbox.RenderTestApp</string>
+  <key>CFBundleInfoDictionaryVersion</key>
+  <string>1.0</string>
+  <key>CFBundleLongVersionString</key>
+  <string>1.0</string>
+  <key>CFBundleName</key>
+  <string>com.mapbox.RenderTestApp</string>
+  <key>CFBundlePackageType</key>
+  <string>APPL</string>
+  <key>CFBundleShortVersionString</key>
+  <string>1.0</string>
+  <key>CFBundleSignature</key>
+  <string>MBGL</string>
+  <key>CFBundleVersion</key>
+  <string>7877</string>
+  <key>CSResourcesFileMapped</key>
+  <true/>
+  <key>NSHumanReadableCopyright</key>
+  <string>© 2014–2020 Mapbox</string>
+  <key>LSMinimumSystemVersion</key>
+  <string>${MACOSX_DEPLOYMENT_TARGET}</string>
+  <key>LSRequiresIPhoneOS</key>
+  <true/>
+  <key>UILaunchStoryboardName</key>
+  <string>LaunchScreen</string>
+  <key>UIMainStoryboardFile</key>
+  <string>Main</string>
+  <key>UIBackgroundModes</key>
+  <array>
+    <string>fetch</string>
+    <string>remote-notification</string>
+  </array>
+  <key>UIRequiredDeviceCapabilities</key>
+  <array>
+    <string>armv7</string>
+  </array>
+  <key>UISupportedInterfaceOrientations</key>
+  <array>
+    <string>UIInterfaceOrientationPortrait</string>
+    <string>UIInterfaceOrientationLandscapeLeft</string>
+    <string>UIInterfaceOrientationLandscapeRight</string>
+  </array>
+  <key>NSAppTransportSecurity</key>
+  <dict>
+    <key>NSAllowsArbitraryLoads</key>
+    <true/>
+  </dict>
+</dict>
+</plist>

render-test/ios/README.md

@@ -0,0 +1,7 @@
+# iOS RenderTestRunner App
+
+This is a blank single-view-controller iOS app, linked against to a C++ static libraries, plus a a simple unit xctest. We use CMake to create an Xcode-friendly out-of-source build system. In another word, the build system is maintained by `CMakeLists` file,instead of a configured `.xcodeproj` file.
+
+This CMake project can build the executable RenderTestApp which is linked to the static C++ library 'mbgl-render-test'. CMakeLists.txt files are the only build configuration kept in source control. This is in contrast to committing the `.xcodeproj` directory which includes the backing XML, which is nonsensically hard to edit by hand.
+
+The test instantiates ObjC object of class `IosTestRunner` from the app, the class will instantiates a C++ object from the linked library `mbgl-render-test` and calls running render test function on it. It subsequently deletes the C++ object pointer. In the end, the test will check the existence of test reports, which are html files that attach to xctest result.

render-test/ios/RenderTestApp_iphoneos13.2-arm64e-debug.xctestrun

@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>RenderTestAppTests</key>
+	<dict>
+		<key>BlueprintName</key>
+		<string>RenderTestAppTests</string>
+		<key>BundleIdentifiersForCrashReportEmphasis</key>
+		<array>
+			<string>com.mapbox.RenderTestApp</string>
+			<string>com.mapbox.RenderTestAppTests</string>
+		</array>
+		<key>CommandLineArguments</key>
+		<array/>
+		<key>DependentProductPaths</key>
+		<array>
+			<string>__TESTROOT__/Debug-iphoneos/RenderTestApp.app</string>
+			<string>__TESTROOT__/Debug-iphoneos/RenderTestApp.app/PlugIns/RenderTestAppTests.xctest</string>
+		</array>
+		<key>EnvironmentVariables</key>
+		<dict>
+			<key>OS_ACTIVITY_DT_MODE</key>
+			<string>YES</string>
+			<key>SQLITE_ENABLE_THREAD_ASSERTIONS</key>
+			<string>1</string>
+		</dict>
+		<key>IsAppHostedTestBundle</key>
+		<true/>
+		<key>ProductModuleName</key>
+		<string>RenderTestAppTests</string>
+		<key>RunOrder</key>
+		<integer>0</integer>
+		<key>SystemAttachmentLifetime</key>
+		<string>deleteOnSuccess</string>
+		<key>TestBundlePath</key>
+		<string>__TESTHOST__/PlugIns/RenderTestAppTests.xctest</string>
+		<key>TestHostBundleIdentifier</key>
+		<string>com.mapbox.RenderTestApp</string>
+		<key>TestHostPath</key>
+		<string>__TESTROOT__/Debug-iphoneos/RenderTestApp.app</string>
+		<key>TestLanguage</key>
+		<string></string>
+		<key>TestRegion</key>
+		<string></string>
+		<key>TestingEnvironmentVariables</key>
+		<dict>
+			<key>DYLD_FALLBACK_FRAMEWORK_PATH</key>
+			<string></string>
+			<key>DYLD_INSERT_LIBRARIES</key>
+			<string>__PLATFORMS__/iPhoneOS.platform/Developer/usr/lib/libXCTestBundleInject.dylib</string>
+			<key>XCInjectBundleInto</key>
+			<string>unused</string>
+		</dict>
+		<key>ToolchainsSettingValue</key>
+		<array/>
+		<key>UITargetAppCommandLineArguments</key>
+		<array/>
+		<key>UserAttachmentLifetime</key>
+		<string>deleteOnSuccess</string>
+	</dict>
+	<key>__xctestrun_metadata__</key>
+	<dict>
+		<key>FormatVersion</key>
+		<integer>1</integer>
+	</dict>
+</dict>
+</plist>

render-test/ios/RenderTestApp_iphoneos13.2-arm64e-release.xctestrun

@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>RenderTestAppTests</key>
+	<dict>
+		<key>BlueprintName</key>
+		<string>RenderTestAppTests</string>
+		<key>BundleIdentifiersForCrashReportEmphasis</key>
+		<array>
+			<string>com.mapbox.RenderTestApp</string>
+			<string>com.mapbox.RenderTestAppTests</string>
+		</array>
+		<key>CommandLineArguments</key>
+		<array/>
+		<key>DependentProductPaths</key>
+		<array>
+			<string>__TESTROOT__/Release-iphoneos/RenderTestApp.app</string>
+			<string>__TESTROOT__/Release-iphoneos/RenderTestApp.app/PlugIns/RenderTestAppTests.xctest</string>
+		</array>
+		<key>EnvironmentVariables</key>
+		<dict>
+			<key>OS_ACTIVITY_DT_MODE</key>
+			<string>YES</string>
+			<key>SQLITE_ENABLE_THREAD_ASSERTIONS</key>
+			<string>1</string>
+		</dict>
+		<key>IsAppHostedTestBundle</key>
+		<true/>
+		<key>ProductModuleName</key>
+		<string>RenderTestAppTests</string>
+		<key>RunOrder</key>
+		<integer>0</integer>
+		<key>SystemAttachmentLifetime</key>
+		<string>deleteOnSuccess</string>
+		<key>TestBundlePath</key>
+		<string>__TESTHOST__/PlugIns/RenderTestAppTests.xctest</string>
+		<key>TestHostBundleIdentifier</key>
+		<string>com.mapbox.RenderTestApp</string>
+		<key>TestHostPath</key>
+		<string>__TESTROOT__/Release-iphoneos/RenderTestApp.app</string>
+		<key>TestLanguage</key>
+		<string></string>
+		<key>TestRegion</key>
+		<string></string>
+		<key>TestingEnvironmentVariables</key>
+		<dict>
+			<key>DYLD_FALLBACK_FRAMEWORK_PATH</key>
+			<string></string>
+			<key>DYLD_INSERT_LIBRARIES</key>
+			<string>__PLATFORMS__/iPhoneOS.platform/Developer/usr/lib/libXCTestBundleInject.dylib</string>
+			<key>XCInjectBundleInto</key>
+			<string>unused</string>
+		</dict>
+		<key>ToolchainsSettingValue</key>
+		<array/>
+		<key>UITargetAppCommandLineArguments</key>
+		<array/>
+		<key>UserAttachmentLifetime</key>
+		<string>deleteOnSuccess</string>
+	</dict>
+	<key>__xctestrun_metadata__</key>
+	<dict>
+		<key>FormatVersion</key>
+		<integer>1</integer>
+	</dict>
+</dict>
+</plist>

render-test/ios/codesigning/RenderTestApp.app.xcent.template

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>application-identifier</key>
+	<string>$TEAM_ID.com.mapbox.RenderTestApp</string>
+	<key>com.apple.developer.team-identifier</key>
+	<string>$TEAM_ID</string>
+	<key>get-task-allow</key>
+	<true/>
+	<key>keychain-access-groups</key>
+	<array>
+		<string>$TEAM_ID.com.mapbox.RenderTestApp</string>
+	</array>
+</dict>
+</plist>

render-test/ios/codesigning/RenderTestAppTests.xctest.xcent.template

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>application-identifier</key>
+	<string>$TEAM_ID.com.mapbox.RenderTestAppTests</string>
+	<key>com.apple.developer.team-identifier</key>
+	<string>$TEAM_ID</string>
+	<key>get-task-allow</key>
+	<true/>
+	<key>keychain-access-groups</key>
+	<array>
+		<string>$TEAM_ID.com.mapbox.RenderTestAppTests</string>
+	</array>
+</dict>
+</plist>

render-test/ios/codesigning/XCTAutomationSupport.framework.xcent.template

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>application-identifier</key>
+	<string>$TEAM_ID.com.mapbox.RenderTestAppTests</string>
+	<key>com.apple.developer.team-identifier</key>
+	<string>$TEAM_ID</string>
+	<key>get-task-allow</key>
+	<true/>
+	<key>keychain-access-groups</key>
+	<array>
+		<string>$TEAM_ID.com.mapbox.RenderTestAppTests</string>
+	</array>
+</dict>
+</plist>

render-test/ios/codesigning/XCTest.framework.xcent.template

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>application-identifier</key>
+	<string>$TEAM_ID.com.mapbox.RenderTestAppTests</string>
+	<key>com.apple.developer.team-identifier</key>
+	<string>$TEAM_ID</string>
+	<key>get-task-allow</key>
+	<true/>
+	<key>keychain-access-groups</key>
+	<array>
+		<string>$TEAM_ID.com.mapbox.RenderTestAppTests</string>
+	</array>
+</dict>
+</plist>

render-test/ios/codesigning/codesign_all.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+identity=$(/usr/bin/env xcrun security find-identity -v -p codesigning)
+var=${identity#*1)\ }
+CODE_SIGN_IDENTITY=${var%\ \"Apple\ Development*}
+
+codesign --force --sign $CODE_SIGN_IDENTITY --entitlements codesigning/RenderTestApp.app.xcent --timestamp=none ./Release-iphoneos/RenderTestApp.app
+codesign --force --sign $CODE_SIGN_IDENTITY --deep --preserve-metadata=identifier,entitlements,flags --timestamp=none ./Release-iphoneos/RenderTestApp.app/Frameworks/libXCTestSwiftSupport.dylib
+codesign --force --sign $CODE_SIGN_IDENTITY --deep --preserve-metadata=identifier,entitlements,flags --timestamp=none ./Release-iphoneos/RenderTestApp.app/Frameworks/libXCTestBundleInject.dylib
+codesign --force --sign $CODE_SIGN_IDENTITY --deep --preserve-metadata=identifier,entitlements,flags --timestamp=none ./Release-iphoneos/RenderTestApp.app/Frameworks/XCTAutomationSupport.framework
+codesign --force --sign $CODE_SIGN_IDENTITY --deep --preserve-metadata=identifier,entitlements,flags --timestamp=none ./Release-iphoneos/RenderTestApp.app/Frameworks/XCTest.framework
+codesign --force --sign $CODE_SIGN_IDENTITY --deep --entitlements codesigning/RenderTestAppTests.xctest.xcent --timestamp=none ./Release-iphoneos/RenderTestApp.app/PlugIns/RenderTestAppTests.xctest

render-test/ios/codesigning/generate-entitlements.swift

@@ -0,0 +1,40 @@
+#!/usr/bin/swift
+
+/// Generates Entitlement files .xcent for codesigning, containing the correct team id
+import Swift
+import Foundation
+
+let fileManager = FileManager.default
+
+let contentsOfCurrentWorkingDirectory = try FileManager.default.contentsOfDirectory(at: URL(fileURLWithPath: "./codesigning/"), includingPropertiesForKeys: nil, options: [])
+
+guard let rawTeamIdEnv = getenv("IOS_DEVELOPMENT_TEAM") else {
+    print("Please add a IOS_DEVELOPMENT_TEAM environment variable.")
+    exit(1)
+}
+
+let teamId2 = String(utf8String: rawTeamIdEnv)
+
+for file in contentsOfCurrentWorkingDirectory {
+    if (file.pathExtension == "template") {
+        var content = ""
+        do {
+            content = try String(contentsOf: file, encoding: .utf8)
+        }
+        catch {
+            print("Error reading xcent file: \(error).")
+        }
+
+        content = content.replacingOccurrences(of: "$TEAM_ID", with: teamId2!)
+
+        var targetFile = file
+        targetFile.deletePathExtension()
+        do {
+            try content.write(to: targetFile, atomically: false, encoding: .utf8)
+            print("Entitlement " + targetFile.absoluteString + " generated.")
+        }
+        catch {
+            print("Error writing xcent file: \(error).")
+        }
+    }
+}

render-test/ios/codesigning/libXCTestSwiftSupport.dylib.xcent.template

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>application-identifier</key>
+	<string>$TEAM_ID.com.mapbox.RenderTestAppTests</string>
+	<key>com.apple.developer.team-identifier</key>
+	<string>$TEAM_ID</string>
+	<key>get-task-allow</key>
+	<true/>
+	<key>keychain-access-groups</key>
+	<array>
+		<string>$TEAM_ID.com.mapbox.RenderTestAppTests</string>
+	</array>
+</dict>
+</plist>

render-test/ios/fastlane/Appfile

@@ -0,0 +1,3 @@
+app_identifier("com.mapbox.RenderTestApp") # The bundle identifier of your app
+apple_id ENV['MAPBOX_APPLE_ID']
+team_id ENV['MAPBOX_APPLE_TEAM_ID'] # Developer Portal Team ID

render-test/ios/fastlane/Matchfile

@@ -0,0 +1,5 @@
+git_url("git@github.com:mapbox/apple-certificates.git")
+type("development") # The default type, can be: appstore, adhoc, enterprise or development
+app_identifier(["com.mapbox.RenderTestApp", "com.mapbox.RenderTestAppTests"])
+username ENV['MAPBOX_APPLE_ID']
+keychain_name("fastlane_keychain")

render-test/ios/iosTestRunner.h

@@ -0,0 +1,11 @@
+#import <Foundation/Foundation.h>
+
+__attribute__((visibility ("default")))
+@interface IosTestRunner : NSObject
+
+- (NSString*) getStyleResultPath;
+- (NSString*) getMetricResultPath;
+- (NSString*) getMetricPath;
+- (BOOL) getTestStatus;
+
+@end

render-test/ios/iosTestRunner.mm

@@ -0,0 +1,157 @@
+#import "iosTestRunner.h"
+
+#include "ios_test_runner.hpp"
+
+#import "SSZipArchive.h"
+
+#include <string>
+
+@interface IosTestRunner ()
+
+@property (nullable) TestRunner* runner;
+
+@property (copy, nullable) NSString *styleResultPath;
+
+@property (copy, nullable) NSString *metricResultPath;
+
+@property (copy, nullable) NSString *metricPath;
+
+@property BOOL testStatus;
+
+@end
+
+@implementation IosTestRunner
+
+-(instancetype)init
+{
+    self = [super init];
+    if (self) {
+        self.testStatus = NO;
+        self.runner = new TestRunner();
+        NSString *path = nil;
+        NSError *error;
+        NSFileManager *fileManager = [NSFileManager defaultManager];
+        NSString *bundleRoot = [[NSBundle mainBundle] bundlePath];
+        NSArray *bundleContents = [fileManager contentsOfDirectoryAtPath: bundleRoot error: &error];
+        NSArray *paths = NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES);
+        NSString *documentsDir = [paths objectAtIndex: 0];
+        
+        for (uint32_t i = 0; i < bundleContents.count; i++) {
+            NSString *dirName = [bundleContents objectAtIndex: i];
+            if ([dirName isEqualToString:@"test-data"]) {
+                NSString *destinationPath = [documentsDir stringByAppendingPathComponent: dirName];
+                //Using NSFileManager we can perform many file system operations.
+                BOOL success = [fileManager fileExistsAtPath: destinationPath];
+                if (success) {
+                    success = [fileManager removeItemAtPath:destinationPath error:NULL];
+                }
+                break;
+            }
+        }
+        
+        for (uint32_t i = 0; i < bundleContents.count; i++) {
+            NSString *dirName = [bundleContents objectAtIndex: i];
+            if ([dirName isEqualToString:@"test-data"]) {
+                NSString *destinationPath = [documentsDir stringByAppendingPathComponent: dirName];
+                BOOL success = [fileManager fileExistsAtPath: destinationPath];
+                if (!success) {
+                    NSString *copyDirPath  = [[[NSBundle mainBundle] resourcePath] stringByAppendingPathComponent: dirName];
+
+                    success = [fileManager copyItemAtPath: copyDirPath toPath: destinationPath error: &error];
+              
+                    if (!success){
+                        NSAssert1(0, @"Failed to copy file '%@'.", [error localizedDescription]);
+                        NSLog(@"Failed to copy %@ file, error %@", dirName, [error localizedDescription]);
+                    }
+                    else {
+                        path = destinationPath;
+                        NSLog(@"File copied %@ OK", dirName);
+                    }
+                }
+                else {
+                    NSLog(@"File exits %@, skip copy", dirName);
+                }
+                break;
+            }
+        }
+        if (path) {
+
+            std::string basePath = std::string([path UTF8String]);
+            self.testStatus = self.runner->startTest(basePath) ? YES : NO;
+            self.styleResultPath =  [path stringByAppendingPathComponent:@"/ios-render-test-runner-style.html"];
+            self.metricResultPath =  [path stringByAppendingPathComponent:@"/ios-render-test-runner-metrics.html"];
+
+            BOOL fileFound = [fileManager fileExistsAtPath: self.styleResultPath];
+            if (fileFound == NO) {
+                NSLog(@"Style test result file '%@' doese not exit ", self.styleResultPath);
+                self.testStatus = NO;
+            }
+
+            fileFound = [fileManager fileExistsAtPath: self.metricResultPath];
+            if (fileFound == NO) {
+                NSLog(@"Metric test result file '%@' doese not exit ", self.metricResultPath);
+                self.testStatus = NO;
+            }
+
+            NSString *rebaselinePath = [path stringByAppendingPathComponent:@"/baselines"];
+            BOOL needArchiving = NO;
+            BOOL isDir = NO;
+            NSArray *subpaths = [[NSArray alloc] init];
+            if ([fileManager fileExistsAtPath:rebaselinePath isDirectory: &isDir] && isDir){
+                subpaths = [fileManager subpathsAtPath:rebaselinePath];
+                for(NSString *path in subpaths)
+                {
+                    NSString *longPath = [rebaselinePath stringByAppendingPathComponent:path];
+                    if([fileManager fileExistsAtPath:longPath isDirectory:&isDir] && !isDir)
+                    {
+                        needArchiving = YES;
+                        break;
+                    }
+                }
+            }
+            else {
+               NSLog(@"Metric path '%@' doese not exit ", rebaselinePath);
+            }
+
+            if (needArchiving) {
+                NSString *archivePath = [path stringByAppendingString:@"/metrics.zip"];
+                BOOL success = [SSZipArchive createZipFileAtPath:archivePath
+                withContentsOfDirectory:rebaselinePath
+                    keepParentDirectory:NO
+                       compressionLevel:-1
+                               password:nil
+                                    AES:YES
+                        progressHandler:nil];
+
+                if (success) {
+                    NSLog(@"Successfully archive all of the metrics into metrics.zip");
+                    self.metricPath =  archivePath;
+                }
+                else {
+                    NSLog(@"Failed to archive rebaselined metrics into metrics.zip");
+                }
+            }
+        }
+
+        delete self.runner;
+        self.runner = nullptr;
+    }
+    return self;
+}
+
+- (NSString*) getStyleResultPath {
+    return self.styleResultPath;
+}
+
+- (NSString*) getMetricResultPath {
+    return self.metricResultPath;
+}
+
+- (NSString*) getMetricPath {
+    return self.metricPath;
+}
+
+- (BOOL) getTestStatus {
+    return self.testStatus;
+}
+@end

render-test/ios/ios_test_runner.cpp

@@ -0,0 +1,49 @@
+#include "ios_test_runner.hpp"
+
+#include <mbgl/render_test.hpp>
+
+#include <mbgl/util/logging.hpp>
+
+#include <vector>
+
+#define EXPORT __attribute__((visibility("default")))
+
+EXPORT
+bool TestRunner::startTest(const std::string& manifestBasePath) {
+    auto runTestWithManifest = [](const std::string& manifest) -> bool {
+        std::vector<std::string> arguments = {"mbgl-render-test-runner", "-p", manifest, "-u", "rebaseline"};
+        std::vector<char*> argv;
+        for (const auto& arg : arguments) {
+            argv.push_back(const_cast<char*>(arg.data()));
+        }
+        argv.push_back(nullptr);
+
+        int finishedTestCount = 0;
+        std::function<void()> testStatus = [&]() {
+            mbgl::Log::Info(mbgl::Event::General, "Current finished tests number is '%d' ", ++finishedTestCount);
+        };
+        mbgl::Log::Info(mbgl::Event::General, "Start running RenderTestRunner with manifest: '%s'", manifest.c_str());
+
+        auto result = mbgl::runRenderTests(static_cast<int>(argv.size() - 1), argv.data(), testStatus);
+
+        mbgl::Log::Info(mbgl::Event::General,
+                        "End running RenderTestRunner with manifest: '%s' with result value %d",
+                        manifest.c_str(),
+                        result);
+        return result == 0;
+    };
+
+    bool status = false;
+    try {
+        status = runTestWithManifest(manifestBasePath + "/ios-render-test-runner-style.json");
+        status = runTestWithManifest(manifestBasePath + "/ios-render-test-runner-metrics.json") && status;
+    } catch (...) {
+        mbgl::Log::Info(mbgl::Event::General, "Failed with exception");
+    }
+
+    mbgl::Log::Info(mbgl::Event::General, "All tests are finished!");
+    if (!status) {
+        mbgl::Log::Info(mbgl::Event::General, "There are failing test cases");
+    }
+    return status;
+}

render-test/ios/setup_test_data.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+if [ $# -eq 0 ]; then
+    echo "Give base directory as argument, eg. 'build'"
+    exit 1
+fi
+
+BASE_DIR=$1
+
+rm -rf $BASE_DIR/test-data/
+
+mkdir -p $BASE_DIR/test-data/integration/
+mkdir -p $BASE_DIR/test-data/baselines/
+
+cp -r mapbox-gl-js/test/integration/render-tests $BASE_DIR/test-data/integration/
+cp -r mapbox-gl-js/test/integration/query-tests $BASE_DIR/test-data/integration/
+cp -r metrics/expectations $BASE_DIR/test-data/
+cp -r metrics/ignores $BASE_DIR/test-data/
+cp -r metrics/tests $BASE_DIR/test-data/
+cp -r metrics/ios-render-test-runner $BASE_DIR/test-data/
+cp -r metrics/ios-render-test-runner-style.json $BASE_DIR/test-data/
+cp -r metrics/ios-render-test-runner-metrics.json $BASE_DIR/test-data/
+cp -r metrics/cache-metrics.db $BASE_DIR/test-data/
+cp -r metrics/cache-style.db $BASE_DIR/test-data/

render-test/ios/tests/Info.plist

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CFBundleDevelopmentRegion</key>
+	<string>en</string>
+	<key>CFBundleExecutable</key>
+	<string>RenderTestAppTests</string>
+	<key>CFBundleIdentifier</key>
+	<string>com.mapbox.RenderTestAppTests</string>
+	<key>CFBundleInfoDictionaryVersion</key>
+	<string>1.0</string>
+	<key>CFBundleName</key>
+	<string>RenderTestAppTests</string>
+	<key>CFBundlePackageType</key>
+	<string>BNDL</string>
+	<key>CFBundleShortVersionString</key>
+	<string>1.0</string>
+	<key>CFBundleVersion</key>
+	<string>1</string>
+</dict>
+</plist>

render-test/ios/tests/Tests.m

@@ -0,0 +1,59 @@
+#import <XCTest/XCTest.h>
+#import "iosTestRunner.h"
+@interface Tests : XCTestCase
+
+@end
+
+@implementation Tests
+
+- (void)setUp {
+    [super setUp];
+}
+
+- (void)tearDown {
+    [super tearDown];
+}
+
+- (void)testStartRenderTestRunner {
+    IosTestRunner* runner = [[IosTestRunner alloc] init]; 
+    XCTAssert(runner, @"IOSTestRunner is not initialized correctly");
+
+    NSString* styleResult = [runner getStyleResultPath];
+    XCTAssert(styleResult, @"IOSTestRunner does not produce a style test result file");
+    NSString* metricResult = [runner getMetricResultPath];
+    XCTAssert(metricResult, @"IOSTestRunner does not produce a metric test result file");
+    
+    NSFileManager *fileManager = [NSFileManager defaultManager];
+    BOOL fileFound = [fileManager fileExistsAtPath: styleResult];
+    XCTAssert(fileFound, @"Test result html '%@' doese not exit", styleResult);
+    NSURL *styleURL = [NSURL fileURLWithPath:styleResult];
+    XCTAttachment *attachment1URL = [XCTAttachment attachmentWithContentsOfFileAtURL: styleURL];
+    XCTAssert(attachment1URL, @"Failed to attach test result '%@'", styleResult);
+    attachment1URL.lifetime = XCTAttachmentLifetimeKeepAlways;
+    [self addAttachment:attachment1URL];
+
+    fileFound = [fileManager fileExistsAtPath: metricResult];
+    XCTAssert(fileFound, @"Test result html '%@' doese not exit", metricResult);
+    NSURL *metricURL = [NSURL fileURLWithPath:metricResult];
+    XCTAttachment *attachment2URL = [XCTAttachment attachmentWithContentsOfFileAtURL: metricURL];
+    XCTAssert(attachment2URL, @"Failed to attach test result '%@'", metricResult);
+    attachment2URL.lifetime = XCTAttachmentLifetimeKeepAlways;
+    [self addAttachment:attachment2URL];
+
+    NSString* metrics = [runner getMetricPath];
+    if (metrics && [fileManager fileExistsAtPath: metrics]) {
+        NSURL *metricsURL = [NSURL fileURLWithPath:metrics];
+        XCTAttachment *attachmentMetricsURL = [XCTAttachment attachmentWithContentsOfFileAtURL: metricsURL];
+        XCTAssert(attachmentMetricsURL, @"Failed to attach test rebaselined metrics '%@'", metrics);
+        attachmentMetricsURL.lifetime = XCTAttachmentLifetimeKeepAlways;
+        [self addAttachment:attachmentMetricsURL];
+    }
+    else {
+        NSLog(@"No rebaselined metrics are found");
+    }
+
+    BOOL success = [runner getTestStatus];
+    XCTAssert(success, @"IOSTestRunner reports error because some of the tests are not passed, please check the test report");
+}
+
+@end

render-test/manifest_parser.cpp

@@ -0,0 +1,293 @@
+#include "manifest_parser.hpp"
+#include "filesystem.hpp"
+#include "parser.hpp"
+
+#include <mbgl/util/logging.hpp>
+
+#include <algorithm>
+#include <random>
+
+Manifest::Manifest() {
+    const char* envToken = getenv("MAPBOX_ACCESS_TOKEN");
+    if (envToken != nullptr) {
+        accessToken = envToken;
+    }
+}
+
+Manifest::~Manifest() = default;
+
+const std::vector<TestPaths>& Manifest::getTestPaths() const {
+    return testPaths;
+}
+const std::vector<std::pair<std::string, std::string>>& Manifest::getIgnores() const {
+    return ignores;
+}
+const std::string& Manifest::getTestRootPath() const {
+    return testRootPath;
+}
+const std::string& Manifest::getManifestPath() const {
+    return manifestPath;
+}
+const std::string& Manifest::getResultPath() const {
+    return resultPath;
+}
+
+const std::string& Manifest::getCachePath() const {
+    return cachePath;
+}
+
+const std::string& Manifest::getAccessToken() const {
+    return accessToken;
+}
+
+const std::set<std::string>& Manifest::getProbes() const {
+    return probes;
+}
+
+void Manifest::doShuffle(uint32_t seed) {
+    std::seed_seq sequence{seed};
+    std::mt19937 shuffler(sequence);
+    std::shuffle(testPaths.begin(), testPaths.end(), shuffler);
+}
+
+namespace {
+std::vector<std::pair<std::string, std::string>> parseIgnores(const std::vector<mbgl::filesystem::path>& ignoresPaths) {
+    std::vector<std::pair<std::string, std::string>> ignores;
+    for (const auto& path : ignoresPaths) {
+        auto maybeIgnores = readJson(path);
+        if (!maybeIgnores.is<mbgl::JSDocument>()) {
+            continue;
+        }
+        for (const auto& property : maybeIgnores.get<mbgl::JSDocument>().GetObject()) {
+            const std::string ignore = {property.name.GetString(), property.name.GetStringLength()};
+            const std::string reason = {property.value.GetString(), property.value.GetStringLength()};
+            ignores.emplace_back(std::make_pair(ignore, reason));
+        }
+    }
+
+    return ignores;
+}
+
+// defaultExpectationPath: absolute path that constains the style.json file for testing
+// testId: Test case id that used for composing expectation path
+// expectatedPaths: absolute paths that constain possible expected.png/metrics.json files for result checking
+std::vector<mbgl::filesystem::path> getTestExpectations(const mbgl::filesystem::path& defaultExpectationPath,
+                                                        const std::string& testId,
+                                                        std::vector<mbgl::filesystem::path> expectatedPaths) {
+    std::vector<mbgl::filesystem::path> expectations{defaultExpectationPath};
+    for (const auto& expectedPath : expectatedPaths) {
+        expectations.emplace_back(expectedPath / testId);
+    }
+    return expectations;
+}
+
+mbgl::filesystem::path getValidPath(const std::string& manifestPath, const std::string& path) {
+    const static mbgl::filesystem::path BasePath{manifestPath};
+    mbgl::filesystem::path result{path};
+    if (result.is_relative()) {
+        result = BasePath / result;
+    }
+    if (mbgl::filesystem::exists(result)) {
+        return result.lexically_normal();
+    }
+    mbgl::Log::Warning(mbgl::Event::General, "Invalid path is provided inside the manifest file: %s", path.c_str());
+    return mbgl::filesystem::path{};
+}
+
+} // namespace
+
+mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& manifestPath,
+                                                       std::string testFilter) {
+    Manifest manifest;
+    const auto filePath = mbgl::filesystem::path(manifestPath);
+    manifest.manifestPath = manifestPath.substr(0, manifestPath.find(filePath.filename()));
+
+    auto contents = readJson(filePath);
+    if (!contents.is<mbgl::JSDocument>()) {
+        mbgl::Log::Error(mbgl::Event::General, "Provided manifest file: %s is not a valid json", filePath.c_str());
+        return mbgl::nullopt;
+    }
+
+    auto document = std::move(contents.get<mbgl::JSDocument>());
+    if (document.HasMember("result_path")) {
+        const auto& resultPathValue = document["result_path"];
+        if (!resultPathValue.IsString()) {
+            mbgl::Log::Warning(
+                mbgl::Event::General, "Invalid result_path is provided inside the manifest file: %s", filePath.c_str());
+            return mbgl::nullopt;
+        }
+        manifest.resultPath = (getValidPath(manifest.manifestPath, resultPathValue.GetString()) / "").string();
+        if (manifest.resultPath.empty()) {
+            return mbgl::nullopt;
+        }
+    }
+    if (document.HasMember("cache_path")) {
+        const auto& cachePathValue = document["cache_path"];
+        if (!cachePathValue.IsString()) {
+            mbgl::Log::Warning(
+                mbgl::Event::General, "Invalid cache_path is provided inside the manifest file: %s", filePath.c_str());
+            return mbgl::nullopt;
+        }
+        manifest.cachePath = (getValidPath(manifest.manifestPath, ".") / cachePathValue.GetString()).string();
+        if (manifest.cachePath.empty()) {
+            return mbgl::nullopt;
+        }
+    }
+    if (document.HasMember("access_token")) {
+        const auto& accessTokenValue = document["access_token"];
+        if (!accessTokenValue.IsString()) {
+            mbgl::Log::Warning(mbgl::Event::General,
+                               "Invalid access_token is provided inside the manifest file: %s",
+                               filePath.c_str());
+            return mbgl::nullopt;
+        }
+        manifest.accessToken = accessTokenValue.GetString();
+        if (manifest.accessToken.empty()) {
+            return mbgl::nullopt;
+        }
+    }
+    mbgl::filesystem::path baseTestPath;
+    if (document.HasMember("base_test_path")) {
+        const auto& testPathValue = document["base_test_path"];
+        if (!testPathValue.IsString()) {
+            mbgl::Log::Warning(mbgl::Event::General,
+                               "Invalid base_test_path is provided inside the manifest file: %s",
+                               filePath.c_str());
+            return mbgl::nullopt;
+        }
+        baseTestPath = getValidPath(manifest.manifestPath, testPathValue.GetString());
+        if (baseTestPath.empty()) {
+            return mbgl::nullopt;
+        }
+    }
+    mbgl::filesystem::path expectedMetricPath;
+    if (document.HasMember("metric_path")) {
+        const auto& metricPathValue = document["metric_path"];
+        if (!metricPathValue.IsString()) {
+            mbgl::Log::Warning(
+                mbgl::Event::General, "Invalid metric_path is provided inside the manifest file: %s", filePath.c_str());
+            return mbgl::nullopt;
+        }
+        expectedMetricPath = getValidPath(manifest.manifestPath, metricPathValue.GetString());
+        if (expectedMetricPath.empty()) {
+            return mbgl::nullopt;
+        }
+    }
+    std::vector<mbgl::filesystem::path> expectationPaths{};
+    if (document.HasMember("expectation_paths")) {
+        const auto& expectationPathValue = document["expectation_paths"];
+        if (!expectationPathValue.IsArray()) {
+            mbgl::Log::Warning(mbgl::Event::General,
+                               "Provided expectation_paths inside the manifest file: %s is not a valid array",
+                               filePath.c_str());
+            return mbgl::nullopt;
+        }
+        for (const auto& value : expectationPathValue.GetArray()) {
+            if (!value.IsString()) {
+                mbgl::Log::Warning(mbgl::Event::General,
+                                   "Invalid expectation path item is provided inside the manifest file: %s",
+                                   filePath.c_str());
+                return mbgl::nullopt;
+            }
+            expectationPaths.emplace_back(getValidPath(manifest.manifestPath, value.GetString()));
+            if (expectationPaths.back().empty()) {
+                return mbgl::nullopt;
+            }
+        }
+    }
+    std::vector<mbgl::filesystem::path> ignorePaths{};
+    if (document.HasMember("ignore_paths")) {
+        const auto& ignorePathValue = document["ignore_paths"];
+        if (!ignorePathValue.IsArray()) {
+            mbgl::Log::Warning(mbgl::Event::General,
+                               "Provided ignore_paths inside the manifest file: %s is not a valid array",
+                               filePath.c_str());
+            return mbgl::nullopt;
+        }
+        for (const auto& value : ignorePathValue.GetArray()) {
+            if (!value.IsString()) {
+                mbgl::Log::Warning(mbgl::Event::General,
+                                   "Invalid ignore path item is provided inside the manifest file: %s",
+                                   filePath.c_str());
+                return mbgl::nullopt;
+            }
+            ignorePaths.emplace_back(getValidPath(manifest.manifestPath, value.GetString()));
+            if (ignorePaths.back().empty()) {
+                return mbgl::nullopt;
+            }
+        }
+        manifest.ignores = parseIgnores(ignorePaths);
+    }
+
+    if (document.HasMember("probes")) {
+        const auto& probesValue = document["probes"];
+        if (!probesValue.IsArray()) {
+            mbgl::Log::Warning(mbgl::Event::General,
+                               "Provided probes inside the manifest file: %s is not a valid array",
+                               filePath.c_str());
+            return mbgl::nullopt;
+        }
+        for (const auto& value : probesValue.GetArray()) {
+            if (!value.IsString()) {
+                mbgl::Log::Warning(mbgl::Event::General,
+                                   "Invalid probe type is provided inside the manifest file: %s",
+                                   filePath.c_str());
+                return mbgl::nullopt;
+            }
+            manifest.probes.emplace(value.GetString());
+        }
+    }
+
+    if (testFilter.empty() && document.HasMember("filter")) {
+        const auto& filterValue = document["filter"];
+        if (!filterValue.IsString()) {
+            mbgl::Log::Warning(
+                mbgl::Event::General, "Invalid filter is provided inside the manifest file: %s", filePath.c_str());
+            return mbgl::nullopt;
+        }
+
+        testFilter = filterValue.GetString();
+    }
+
+    manifest.testRootPath = baseTestPath.string();
+    if (manifest.testRootPath.back() == '/') {
+        manifest.testRootPath.pop_back();
+    }
+    if (manifest.manifestPath.back() == '/') {
+        manifest.manifestPath.pop_back();
+    }
+    if (manifest.resultPath.empty()) {
+        manifest.resultPath = manifest.manifestPath;
+    } else if (manifest.resultPath.back() == '/') {
+        manifest.resultPath.pop_back();
+    }
+
+    auto& path = manifest.testRootPath;
+    auto& testPaths = manifest.testPaths;
+
+    for (auto& testPath : mbgl::filesystem::recursive_directory_iterator(path)) {
+        // Skip paths that fail regexp search.
+        if (!testFilter.empty() && !std::regex_search(testPath.path().string(), std::regex(testFilter))) {
+            continue;
+        }
+
+        if (testPath.path().filename() == "style.json") {
+            const auto defaultExpectationPath{std::move(mbgl::filesystem::path(testPath).remove_filename())};
+            const auto rootLength = manifest.testRootPath.length();
+            auto testId = defaultExpectationPath.string();
+            testId = testId.substr(rootLength + 1, testId.length() - rootLength - 1);
+
+            std::vector<mbgl::filesystem::path> expectedMetricPaths{expectedMetricPath};
+#if defined(__ANDROID__)
+            expectedMetricPaths.emplace_back("/sdcard/baselines/");
+#elif defined(__APPLE__)
+            expectedMetricPaths.emplace_back(manifest.manifestPath + "/baselines/");
+#endif
+            testPaths.emplace_back(testPath,
+                                   getTestExpectations(defaultExpectationPath, testId, expectationPaths),
+                                   getTestExpectations(defaultExpectationPath, testId, expectedMetricPaths));
+        }
+    }
+
+    return mbgl::optional<Manifest>(manifest);
+}

render-test/manifest_parser.hpp

@@ -0,0 +1,44 @@
+#pragma once
+
+#include "metadata.hpp"
+
+#include <mbgl/util/optional.hpp>
+#include <mbgl/util/rapidjson.hpp>
+
+#include <cstdlib>
+#include <regex>
+#include <string>
+#include <utility>
+#include <vector>
+
+class Manifest {
+public:
+    Manifest();
+    ~Manifest();
+    const std::vector<std::pair<std::string, std::string>>& getIgnores() const;
+    const std::vector<TestPaths>& getTestPaths() const;
+    const std::string& getTestRootPath() const;
+    const std::string& getManifestPath() const;
+    const std::string& getResultPath() const;
+    const std::string& getCachePath() const;
+    const std::string& getAccessToken() const;
+    const std::set<std::string>& getProbes() const;
+    void doShuffle(uint32_t seed);
+
+private:
+    friend class ManifestParser;
+    std::string manifestPath;
+    std::string testRootPath;
+    std::string resultPath;
+    std::string cachePath;
+    std::string accessToken;
+    std::vector<std::pair<std::string, std::string>> ignores;
+    std::vector<TestPaths> testPaths;
+    std::set<std::string> probes;
+};
+
+class ManifestParser {
+public:
+    static mbgl::optional<Manifest> parseManifest(const std::string& manifestPath,
+                                                  std::string testFilter);
+};

render-test/metadata.hpp

@@ -0,0 +1,203 @@
+#pragma once
+
+#include <mbgl/map/mode.hpp>
+#include <mbgl/renderer/query.hpp>
+#include <mbgl/storage/file_source.hpp>
+#include <mbgl/util/geo.hpp>
+#include <mbgl/util/rapidjson.hpp>
+#include <mbgl/util/size.hpp>
+
+#include "filesystem.hpp"
+
+#include <list>
+#include <map>
+
+namespace mbgl {
+
+class Map;
+class HeadlessFrontend;
+namespace gfx {
+struct RenderingStats;
+}
+} // namespace mbgl
+
+class TestRunnerMapObserver;
+struct TestStatistics {
+    TestStatistics() = default;
+
+    uint32_t ignoreFailedTests = 0;
+    uint32_t ignorePassedTests = 0;
+    uint32_t erroredTests = 0;
+    uint32_t failedTests = 0;
+    uint32_t passedTests = 0;
+};
+
+struct TestPaths {
+    TestPaths() = default;
+    TestPaths(mbgl::filesystem::path stylePath_,
+              std::vector<mbgl::filesystem::path> expectations_,
+              std::vector<mbgl::filesystem::path> expectedMetrics_)
+        : stylePath(std::move(stylePath_)),
+          expectations(std::move(expectations_)),
+          expectedMetrics(std::move(expectedMetrics_)) {}
+
+    mbgl::filesystem::path stylePath;
+    std::vector<mbgl::filesystem::path> expectations;
+    std::vector<mbgl::filesystem::path> expectedMetrics;
+
+    std::string defaultExpectations() const {
+        assert(!expectations.empty());
+        return expectations.front().string();
+    }
+};
+
+inline std::tuple<bool, float> checkValue(float expected, float actual, float tolerance) {
+    float delta = expected * tolerance;
+    assert(delta >= 0.0f);
+    return std::make_tuple(std::abs(expected - actual) <= delta, delta);
+}
+
+struct FileSizeProbe {
+    FileSizeProbe() = default;
+    FileSizeProbe(std::string path_, uintmax_t size_, float tolerance_)
+        : path(std::move(path_)), size(size_), tolerance(tolerance_) {}
+
+    std::string path;
+    uintmax_t size;
+    float tolerance;
+};
+
+struct MemoryProbe {
+    MemoryProbe() = default;
+    MemoryProbe(size_t peak_, size_t allocations_) : peak(peak_), allocations(allocations_), tolerance(0.0f) {}
+
+    size_t peak;
+    size_t allocations;
+    float tolerance;
+
+    static std::tuple<bool, float> checkPeak(const MemoryProbe& expected, const MemoryProbe& actual) {
+        return checkValue(expected.peak, actual.peak, actual.tolerance);
+    }
+
+    static std::tuple<bool, float> checkAllocations(const MemoryProbe& expected, const MemoryProbe& actual) {
+        return checkValue(expected.allocations, actual.allocations, actual.tolerance);
+    }
+};
+
+struct FpsProbe {
+    float average = 0.0;
+    float minOnePc = 0.0;
+    float tolerance = 0.0f;
+};
+
+struct NetworkProbe {
+    NetworkProbe() = default;
+    NetworkProbe(size_t requests_, size_t transferred_) : requests(requests_), transferred(transferred_) {}
+
+    size_t requests;
+    size_t transferred;
+};
+
+struct GfxProbe {
+    struct Memory {
+        Memory() = default;
+        Memory(int allocated_, int peak_) : allocated(allocated_), peak(peak_) {}
+
+        int allocated;
+        int peak;
+    };
+
+    GfxProbe() = default;
+    GfxProbe(const mbgl::gfx::RenderingStats&, const GfxProbe&);
+
+    int numBuffers;
+    int numDrawCalls;
+    int numFrameBuffers;
+    int numTextures;
+
+    Memory memIndexBuffers;
+    Memory memVertexBuffers;
+    Memory memTextures;
+};
+
+class TestMetrics {
+public:
+    bool isEmpty() const { return fileSize.empty() && memory.empty() && network.empty() && fps.empty() && gfx.empty(); }
+    std::map<std::string, FileSizeProbe> fileSize;
+    std::map<std::string, MemoryProbe> memory;
+    std::map<std::string, NetworkProbe> network;
+    std::map<std::string, FpsProbe> fps;
+    std::map<std::string, GfxProbe> gfx;
+};
+
+struct TestMetadata {
+    TestMetadata() = default;
+
+    TestPaths paths;
+    mbgl::JSDocument document;
+    bool renderTest = true;
+    bool outputsImage = true;
+    bool ignoredTest = false;
+
+    mbgl::Size size{ 512u, 512u };
+    float pixelRatio = 1.0f;
+    double allowed = 0.00015; // diff
+    std::string description;
+    mbgl::MapMode mapMode = mbgl::MapMode::Static;
+    mbgl::MapDebugOptions debug = mbgl::MapDebugOptions::NoDebug;
+    bool crossSourceCollisions = true;
+    bool axonometric = false;
+    double xSkew = 0.0;
+    double ySkew = 1.0;
+    mbgl::ScreenCoordinate queryGeometry{0u, 0u};
+    mbgl::ScreenBox queryGeometryBox{{0u, 0u}, {0u, 0u}};
+    mbgl::RenderedQueryOptions queryOptions;
+
+    // TODO
+    uint32_t fadeDuration = 0;
+    bool addFakeCanvas = false;
+
+    // HTML
+    std::string id;
+    std::string status;
+    std::string color;
+
+    std::string actual;
+    std::string actualJson;
+    std::string expected;
+    std::string diff;
+
+    TestMetrics metrics;
+    TestMetrics expectedMetrics;
+
+    // Results
+    unsigned metricsErrored = 0;
+    unsigned metricsFailed = 0;
+    unsigned renderErrored = 0;
+    unsigned renderFailed = 0;
+    unsigned labelCutOffFound = 0;
+    unsigned duplicationsCount = 0;
+
+    std::string errorMessage;
+    double difference = 0.0;
+
+};
+
+class TestContext {
+public:
+    virtual mbgl::HeadlessFrontend& getFrontend() = 0;
+    virtual mbgl::Map& getMap() = 0;
+    virtual mbgl::FileSource& getFileSource() = 0;
+    virtual TestRunnerMapObserver& getObserver() = 0;
+    virtual TestMetadata& getMetadata() = 0;
+
+    GfxProbe activeGfxProbe{};
+    GfxProbe baselineGfxProbe{};
+    bool gfxProbeActive = false;
+
+protected:
+    virtual ~TestContext() = default;
+};
+
+using TestOperation = std::function<bool(TestContext&)>;
+using TestOperations = std::list<TestOperation>;

render-test/parser.hpp

@@ -0,0 +1,69 @@
+#pragma once
+
+#include "metadata.hpp"
+
+#include <mbgl/util/rapidjson.hpp>
+#include <mbgl/util/variant.hpp>
+
+#include <string>
+#include <tuple>
+#include <vector>
+
+class Manifest;
+
+using ErrorMessage = std::string;
+using JSONReply = mbgl::variant<mbgl::JSDocument, ErrorMessage>;
+
+JSONReply readJson(const mbgl::filesystem::path&);
+std::string serializeJsonValue(const mbgl::JSValue&);
+std::string serializeMetrics(const TestMetrics&);
+
+std::vector<std::string> readExpectedImageEntries(const mbgl::filesystem::path& base);
+std::vector<std::string> readExpectedMetricEntries(const mbgl::filesystem::path& base);
+std::vector<std::string> readExpectedJSONEntries(const mbgl::filesystem::path& base);
+
+TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path);
+
+TestMetadata parseTestMetadata(const TestPaths& paths);
+TestOperations parseTestOperations(TestMetadata& metadata);
+
+std::string createResultPage(const TestStatistics&, const std::vector<TestMetadata>&, bool shuffle, uint32_t seed);
+
+std::string toJSON(const mbgl::Value& value, unsigned indent, bool singleLine);
+std::string toJSON(const std::vector<mbgl::Feature>& features, unsigned indent, bool singleLine);
+
+namespace TestOperationNames {
+extern const std::string waitOp;
+extern const std::string sleepOp;
+extern const std::string addImageOp;
+extern const std::string updateImageOp;
+extern const std::string removeImageOp;
+extern const std::string setStyleOp;
+extern const std::string setCenterOp;
+extern const std::string setZoomOp;
+extern const std::string setBearingOp;
+extern const std::string setPitchOp;
+extern const std::string setFilterOp;
+extern const std::string setLayerZoomRangeOp;
+extern const std::string setLightOp;
+extern const std::string addLayerOp;
+extern const std::string removeLayerOp;
+extern const std::string addSourceOp;
+extern const std::string removeSourceOp;
+extern const std::string setPaintPropertyOp;
+extern const std::string setLayoutPropertyOp;
+extern const std::string fileSizeProbeOp;
+extern const std::string memoryProbeOp;
+extern const std::string memoryProbeStartOp;
+extern const std::string memoryProbeEndOp;
+extern const std::string networkProbeOp;
+extern const std::string networkProbeStartOp;
+extern const std::string networkProbeEndOp;
+extern const std::string setFeatureStateOp;
+extern const std::string getFeatureStateOp;
+extern const std::string removeFeatureStateOp;
+extern const std::string panGestureOp;
+extern const std::string gfxProbeOp;
+extern const std::string gfxProbeStartOp;
+extern const std::string gfxProbeEndOp;
+} // namespace TestOperationNames

render-test/render_test.cpp

@@ -0,0 +1,273 @@
+#include "allocation_index.hpp"
+
+#include <mbgl/render_test.hpp>
+#include <mbgl/storage/network_status.hpp>
+#include <mbgl/util/io.hpp>
+#include <mbgl/util/logging.hpp>
+#include <mbgl/util/run_loop.hpp>
+
+#include <args.hxx>
+
+#include "file_source.hpp"
+#include "manifest_parser.hpp"
+#include "metadata.hpp"
+#include "parser.hpp"
+#include "runner.hpp"
+
+#define ANSI_COLOR_RED        "\x1b[31m"
+#define ANSI_COLOR_GREEN      "\x1b[32m"
+#define ANSI_COLOR_YELLOW     "\x1b[33m"
+#define ANSI_COLOR_BLUE       "\x1b[34m"
+#define ANSI_COLOR_MAGENTA    "\x1b[35m"
+#define ANSI_COLOR_CYAN       "\x1b[36m"
+#define ANSI_COLOR_GRAY       "\x1b[37m"
+#define ANSI_COLOR_LIGHT_GRAY "\x1b[90m"
+#define ANSI_COLOR_RESET      "\x1b[0m"
+
+#if !defined(SANITIZE)
+void* operator new(std::size_t sz) {
+    void* ptr = AllocationIndex::allocate(sz);
+    if (!ptr) throw std::bad_alloc{};
+
+    return ptr;
+}
+
+void operator delete(void* ptr) noexcept {
+    AllocationIndex::deallocate(ptr);
+}
+
+void operator delete(void* ptr, size_t) noexcept {
+    AllocationIndex::deallocate(ptr);
+}
+#endif
+
+namespace {
+
+using ArgumentsTuple = std::tuple<bool, bool, bool, uint32_t, std::string, TestRunner::UpdateResults, std::string>;
+ArgumentsTuple parseArguments(int argc, char** argv) {
+    const static std::unordered_map<std::string, TestRunner::UpdateResults> updateResultsFlags = {
+        {"default", TestRunner::UpdateResults::DEFAULT},
+        {"platform", TestRunner::UpdateResults::PLATFORM},
+        {"metrics", TestRunner::UpdateResults::METRICS},
+        {"rebaseline", TestRunner::UpdateResults::REBASELINE}};
+
+    args::ArgumentParser argumentParser("Mapbox GL Test Runner");
+
+    args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", {'h', "help"});
+
+    args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object", {'r', "recycle-map"});
+    args::Flag shuffleFlag(argumentParser, "shuffle", "Toggle shuffling the tests order", {'s', "shuffle"});
+    args::Flag onlineFlag(
+        argumentParser, "online", "Toggle online mode (by default tests will run offline)", {'o', "online"});
+    args::ValueFlag<uint32_t> seedValue(argumentParser, "seed", "Shuffle seed (default: random)", {"seed"});
+    args::ValueFlag<std::string> testPathValue(
+        argumentParser, "manifestPath", "Test manifest file path", {'p', "manifestPath"}, args::Options::Required);
+    args::ValueFlag<std::string> testFilterValue(argumentParser, "filter", "Test filter regex", {'f', "filter"});
+    args::MapFlag<std::string, TestRunner::UpdateResults> testUpdateResultsValue(
+        argumentParser,
+        "update",
+        "Test results update mode. Supported values are: \n\"default\" Updates generic render test expectations. \
+                                                         \n\"platform\" Updates platform specific render test expectations. \
+                                                         \n\"metrics\" Updates expected metrics for configuration defined by a manifest.\
+                                                         \n\"rebaseline\" Updates or creates expected metrics for configuration defined by a manifest.",
+        {'u', "update"},
+        updateResultsFlags);
+
+    try {
+        argumentParser.ParseCLI(argc, argv);
+    } catch (const args::Help&) {
+        std::ostringstream stream;
+        stream << argumentParser;
+        mbgl::Log::Info(mbgl::Event::General, stream.str());
+        exit(0);
+    } catch (const args::ParseError& e) {
+        std::ostringstream stream;
+        stream << argumentParser;
+        mbgl::Log::Info(mbgl::Event::General, stream.str());
+        mbgl::Log::Error(mbgl::Event::General, e.what());
+        exit(1);
+    } catch (const args::ValidationError& e) {
+        std::ostringstream stream;
+        stream << argumentParser;
+        mbgl::Log::Info(mbgl::Event::General, stream.str());
+        mbgl::Log::Error(mbgl::Event::General, e.what());
+        exit(2);
+    }
+
+    mbgl::filesystem::path manifestPath = args::get(testPathValue);
+    if (!mbgl::filesystem::exists(manifestPath) || !manifestPath.has_filename()) {
+        mbgl::Log::Error(mbgl::Event::General,
+                         "Provided test manifest file path '%s' does not exist",
+                         manifestPath.string().c_str());
+        exit(3);
+    }
+
+    auto testFilter = testFilterValue ? args::get(testFilterValue) : std::string{};
+    const auto shuffle = shuffleFlag ? args::get(shuffleFlag) : false;
+    const auto online = onlineFlag ? args::get(onlineFlag) : false;
+    const auto seed = seedValue ? args::get(seedValue) : 1u;
+    TestRunner::UpdateResults updateResults =
+        testUpdateResultsValue ? args::get(testUpdateResultsValue) : TestRunner::UpdateResults::NO;
+    return ArgumentsTuple{recycleMapFlag ? args::get(recycleMapFlag) : false,
+                          shuffle,
+                          online,
+                          seed,
+                          manifestPath.string(),
+                          updateResults,
+                          std::move(testFilter)};
+}
+} // namespace
+namespace mbgl {
+
+int runRenderTests(int argc, char** argv, std::function<void()> testStatus) {
+    int returnCode = 0;
+    bool recycleMap;
+    bool shuffle;
+    bool online;
+    uint32_t seed;
+    std::string manifestPath;
+    std::string testFilter;
+
+    Log::useLogThread(false);
+    TestRunner::UpdateResults updateResults;
+
+    std::tie(recycleMap, shuffle, online, seed, manifestPath, updateResults, testFilter) = parseArguments(argc, argv);
+
+    ProxyFileSource::setOffline(!online);
+
+    auto manifestData = ManifestParser::parseManifest(manifestPath, testFilter);
+    if (!manifestData) {
+        exit(5);
+    }
+    mbgl::util::RunLoop runLoop;
+    TestRunner runner(std::move(*manifestData), updateResults);
+    if (shuffle) {
+        printf(ANSI_COLOR_YELLOW "Shuffle seed: %d" ANSI_COLOR_RESET "\n", seed);
+        runner.doShuffle(seed);
+    }
+
+    NetworkStatus::Set(online ? NetworkStatus::Status::Online : NetworkStatus::Status::Offline);
+
+    const auto& manifest = runner.getManifest();
+    const auto& ignores = manifest.getIgnores();
+    const auto& testPaths = manifest.getTestPaths();
+    std::vector<TestMetadata> metadatas;
+    metadatas.reserve(testPaths.size());
+
+    TestStatistics stats;
+
+    for (auto& testPath : testPaths) {
+        TestMetadata metadata = parseTestMetadata(testPath);
+
+        if (!recycleMap) {
+            runner.reset();
+        }
+
+        std::string& id = metadata.id;
+        std::string& status = metadata.status;
+        std::string& color = metadata.color;
+
+        const std::string::size_type rootLength = manifest.getTestRootPath().length();
+        id = testPath.defaultExpectations();
+        id = id.substr(rootLength + 1, id.length() - rootLength - 2);
+
+        bool shouldIgnore = false;
+        std::string ignoreReason;
+
+        const std::string ignoreName = id;
+        const auto it = std::find_if(ignores.cbegin(), ignores.cend(), [&ignoreName](auto pair) { return pair.first == ignoreName; });
+        if (it != ignores.end()) {
+            metadata.ignoredTest = shouldIgnore = true;
+            ignoreReason = it->second;
+            if (ignoreReason.rfind("skip", 0) == 0) {
+                printf(ANSI_COLOR_GRAY "* skipped %s (%s)" ANSI_COLOR_RESET "\n", id.c_str(), ignoreReason.c_str());
+                continue;
+            }
+        }
+
+        if (metadata.document.ObjectEmpty()) {
+            metadata.metricsErrored++;
+            metadata.renderErrored++;
+        } else {
+            runner.run(metadata);
+        }
+
+        bool errored = metadata.metricsErrored || metadata.renderErrored || metadata.labelCutOffFound;
+        bool passed = !errored && !metadata.metricsFailed && !metadata.renderFailed;
+
+        if (shouldIgnore) {
+            if (passed) {
+                status = "ignored passed";
+                color = "#E8A408";
+                stats.ignorePassedTests++;
+                printf(ANSI_COLOR_YELLOW "* ignore %s (%s)" ANSI_COLOR_RESET "\n", id.c_str(), ignoreReason.c_str());
+            } else {
+                status = "ignored failed";
+                color = "#9E9E9E";
+                stats.ignoreFailedTests++;
+                printf(ANSI_COLOR_LIGHT_GRAY "* ignore %s (%s)" ANSI_COLOR_RESET "\n", id.c_str(), ignoreReason.c_str());
+            }
+        } else {
+            // Only fail the bots on render errors, this is a CI limitation that we need
+            // to succeed on metrics failed so the rebaseline bot can run next in the
+            // pipeline and collect the new baselines. The rebaseline bot will ultimately
+            // report the error and block the patch from being merged.
+            if (metadata.renderErrored || metadata.renderFailed) {
+                returnCode = 1;
+            }
+
+            if (passed) {
+                status = "passed";
+                color = "green";
+                stats.passedTests++;
+                printf(ANSI_COLOR_GREEN "* passed %s" ANSI_COLOR_RESET "\n", id.c_str());
+            } else if (errored) {
+                status = "errored";
+                color = "red";
+                stats.erroredTests++;
+                printf(ANSI_COLOR_RED "* errored %s" ANSI_COLOR_RESET "\n", id.c_str());
+                printf(ANSI_COLOR_RED "* error: %s" ANSI_COLOR_RESET "\n", metadata.errorMessage.c_str());
+            } else {
+                status = "failed";
+                color = "red";
+                stats.failedTests++;
+                printf(ANSI_COLOR_RED "* failed %s" ANSI_COLOR_RESET "\n", id.c_str());
+            }
+        }
+
+        metadatas.push_back(std::move(metadata));
+        if (testStatus) {
+            testStatus();
+        }
+    }
+
+    const std::string manifestName = mbgl::filesystem::path(manifestPath).stem();
+    const std::string resultPath = manifest.getResultPath() + "/" + manifestName + ".html";
+    std::string resultsHTML = createResultPage(stats, metadatas, shuffle, seed);
+    mbgl::util::write_file(resultPath, resultsHTML);
+
+    const uint32_t count =
+        stats.erroredTests + stats.failedTests + stats.ignoreFailedTests + stats.ignorePassedTests + stats.passedTests;
+
+    if (stats.passedTests) {
+        printf(ANSI_COLOR_GREEN "%u passed (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.passedTests, 100.0 * stats.passedTests / count);
+    }
+    if (stats.ignorePassedTests) {
+        printf(ANSI_COLOR_YELLOW "%u passed but were ignored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.ignorePassedTests, 100.0 * stats.ignorePassedTests / count);
+    }
+    if (stats.ignoreFailedTests) {
+        printf(ANSI_COLOR_LIGHT_GRAY "%u ignored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.ignoreFailedTests, 100.0 * stats.ignoreFailedTests / count);
+    }
+    if (stats.failedTests) {
+        printf(ANSI_COLOR_RED "%u failed (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.failedTests, 100.0 * stats.failedTests / count);
+    }
+    if (stats.erroredTests) {
+        printf(ANSI_COLOR_RED "%u errored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.erroredTests, 100.0 * stats.erroredTests / count);
+    }
+
+    printf("Results at: %s\n", mbgl::filesystem::canonical(resultPath).c_str());
+
+    return returnCode;
+}
+
+} // namespace mbgl

render-test/runner.cpp

@@ -0,0 +1,909 @@
+#include <mbgl/map/camera.hpp>
+#include <mbgl/map/map_observer.hpp>
+#include <mbgl/renderer/renderer.hpp>
+#include <mbgl/renderer/renderer_observer.hpp>
+#include <mbgl/storage/file_source_manager.hpp>
+#include <mbgl/style/image.hpp>
+#include <mbgl/style/layer.hpp>
+#include <mbgl/style/light.hpp>
+#include <mbgl/style/rapidjson_conversion.hpp>
+#include <mbgl/style/style.hpp>
+#include <mbgl/util/chrono.hpp>
+#include <mbgl/util/image.hpp>
+#include <mbgl/util/io.hpp>
+#include <mbgl/util/projection.hpp>
+#include <mbgl/util/run_loop.hpp>
+#include <mbgl/util/string.hpp>
+#include <mbgl/util/tile_cover.hpp>
+
+#include <mapbox/pixelmatch.hpp>
+
+#include <../expression-test/test_runner_common.hpp>
+#include "allocation_index.hpp"
+#include "file_source.hpp"
+#include "metadata.hpp"
+#include "parser.hpp"
+#include "runner.hpp"
+
+#include <algorithm>
+#include <cassert>
+#include <utility>
+#include <sstream>
+
+#include <codecvt>
+#include <locale>
+
+using namespace mbgl;
+using namespace TestOperationNames;
+
+GfxProbe::GfxProbe(const mbgl::gfx::RenderingStats& stats, const GfxProbe& prev)
+    : numBuffers(stats.numBuffers),
+      numDrawCalls(stats.numDrawCalls),
+      numFrameBuffers(stats.numFrameBuffers),
+      numTextures(stats.numActiveTextures),
+      memIndexBuffers(stats.memIndexBuffers, std::max(stats.memIndexBuffers, prev.memIndexBuffers.peak)),
+      memVertexBuffers(stats.memVertexBuffers, std::max(stats.memVertexBuffers, prev.memVertexBuffers.peak)),
+      memTextures(stats.memTextures, std::max(stats.memTextures, prev.memTextures.peak)) {}
+
+// static
+gfx::HeadlessBackend::SwapBehaviour swapBehavior(MapMode mode) {
+    return mode == MapMode::Continuous ? gfx::HeadlessBackend::SwapBehaviour::Flush
+                                       : gfx::HeadlessBackend::SwapBehaviour::NoFlush;
+}
+
+std::string simpleDiff(const Value& result, const Value& expected) {
+    std::vector<std::string> resultTokens{tokenize(toJSON(result, 2, false))};
+    std::vector<std::string> expectedTokens{tokenize(toJSON(expected, 2, false))};
+    std::size_t maxLength = std::max(resultTokens.size(), expectedTokens.size());
+    std::ostringstream diff;
+
+    diff << "<pre>" << std::endl;
+    const auto flush =
+        [](const std::vector<std::string>& vec, std::size_t pos, std::ostringstream& out, std::string separator) {
+            for (std::size_t j = pos; j < vec.size(); ++j) {
+                out << separator << vec[j] << std::endl;
+            }
+        };
+
+    for (std::size_t i = 0; i < maxLength; ++i) {
+        if (resultTokens.size() <= i) {
+            flush(expectedTokens, i, diff, "-");
+            break;
+        }
+
+        if (expectedTokens.size() <= i) {
+            flush(resultTokens, i, diff, "+");
+            break;
+        }
+
+        if (!deepEqual(resultTokens[i], expectedTokens[i])) {
+            diff << "<b>"
+                 << "-" << expectedTokens[i] << "</b>" << std::endl;
+            diff << "<b>"
+                 << "+" << resultTokens[i] << "</b>" << std::endl;
+        } else {
+            diff << resultTokens[i] << std::endl;
+        }
+    }
+    diff << "</pre>" << std::endl;
+    return diff.str();
+}
+
+TestRunner::TestRunner(Manifest manifest_, UpdateResults updateResults_)
+    : manifest(std::move(manifest_)), updateResults(updateResults_) {
+    registerProxyFileSource();
+}
+
+void TestRunner::registerProxyFileSource() {
+    static std::once_flag registerProxyFlag;
+    std::call_once(registerProxyFlag, [] {
+        auto* fileSourceManager = mbgl::FileSourceManager::get();
+
+        auto resourceLoaderFactory =
+            fileSourceManager->unRegisterFileSourceFactory(mbgl::FileSourceType::ResourceLoader);
+        auto factory = [defaultFactory = std::move(resourceLoaderFactory)](const mbgl::ResourceOptions& options) {
+            assert(defaultFactory);
+            std::shared_ptr<FileSource> fileSource = defaultFactory(options);
+            return std::make_unique<ProxyFileSource>(std::move(fileSource), options);
+        };
+
+        fileSourceManager->registerFileSourceFactory(mbgl::FileSourceType::ResourceLoader, std::move(factory));
+    });
+}
+
+const Manifest& TestRunner::getManifest() const {
+    return manifest;
+}
+
+void TestRunner::doShuffle(uint32_t seed) {
+    manifest.doShuffle(seed);
+}
+
+void TestRunner::checkQueryTestResults(mbgl::PremultipliedImage&& actualImage,
+                                       std::vector<mbgl::Feature>&& features,
+                                       TestMetadata& metadata) {
+    const std::string& base = metadata.paths.defaultExpectations();
+    const std::vector<mbgl::filesystem::path>& expectations = metadata.paths.expectations;
+
+    metadata.actual = mbgl::encodePNG(actualImage);
+
+    if (actualImage.size.isEmpty()) {
+        metadata.errorMessage = "Invalid size for actual image";
+        metadata.renderErrored++;
+        return;
+    }
+
+    metadata.actualJson = toJSON(features, 2, false);
+
+    if (metadata.actualJson.empty()) {
+        metadata.errorMessage = "Invalid size for actual JSON";
+        metadata.renderErrored++;
+        return;
+    }
+
+    if (updateResults == UpdateResults::PLATFORM) {
+        mbgl::filesystem::create_directories(expectations.back());
+        mbgl::util::write_file(expectations.back().string() + "/expected.json", metadata.actualJson);
+        metadata.renderErrored++;
+        return;
+    } else if (updateResults == UpdateResults::DEFAULT) {
+        mbgl::util::write_file(base + "/expected.json", metadata.actualJson);
+        metadata.renderErrored++;
+        return;
+    }
+
+    mbgl::util::write_file(base + "/actual.json", metadata.actualJson);
+
+    std::vector<std::string> expectedJsonPaths;
+    mbgl::filesystem::path expectedMetricsPath;
+    for (auto rit = expectations.rbegin(); rit != expectations.rend(); ++rit) {
+        if (mbgl::filesystem::exists(*rit)) {
+            expectedJsonPaths = readExpectedJSONEntries(*rit);
+            if (!expectedJsonPaths.empty()) break;
+        }
+    }
+
+    if (expectedJsonPaths.empty()) {
+        metadata.errorMessage = "Failed to find expectations for: " + metadata.paths.stylePath.string();
+        metadata.renderErrored++;
+        return;
+    }
+
+    for (const auto& entry : expectedJsonPaths) {
+        auto maybeExpectedJson = readJson(entry);
+        if (maybeExpectedJson.is<mbgl::JSDocument>()) {
+            auto& expected = maybeExpectedJson.get<mbgl::JSDocument>();
+
+            mbgl::JSDocument actual;
+            actual.Parse<0>(metadata.actualJson);
+            if (actual.HasParseError()) {
+                metadata.errorMessage = "Error parsing actual JSON for: " + metadata.paths.stylePath.string();
+                metadata.renderErrored++;
+                return;
+            }
+
+            auto actualVal = mapbox::geojson::convert<mapbox::geojson::value>(actual);
+            auto expectedVal = mapbox::geojson::convert<mapbox::geojson::value>(expected);
+            bool equal = deepEqual(actualVal, expectedVal);
+
+            metadata.difference = !equal;
+            if (equal) {
+                metadata.diff = "Match";
+            } else {
+                metadata.diff = simpleDiff(actualVal, expectedVal);
+                metadata.renderFailed++;
+            }
+        } else {
+            metadata.errorMessage = "Failed to load expected JSON " + entry;
+            metadata.renderErrored++;
+            return;
+        }
+    }
+}
+
+void TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage, TestMetadata& metadata) {
+    const std::string& base = metadata.paths.defaultExpectations();
+    const std::vector<mbgl::filesystem::path>& expectations = metadata.paths.expectations;
+
+    if (metadata.outputsImage) {
+        metadata.actual = mbgl::encodePNG(actualImage);
+
+        if (actualImage.size.isEmpty()) {
+            metadata.errorMessage = "Invalid size for actual image";
+            metadata.renderErrored++;
+            return;
+        }
+
+        if (updateResults == UpdateResults::PLATFORM) {
+            mbgl::filesystem::create_directories(expectations.back());
+            mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actualImage));
+            metadata.renderErrored++;
+            return;
+        } else if (updateResults == UpdateResults::DEFAULT) {
+            mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actualImage));
+            metadata.renderErrored++;
+            return;
+        }
+
+        mbgl::util::write_file(base + "/actual.png", metadata.actual);
+
+        mbgl::PremultipliedImage expectedImage{actualImage.size};
+        mbgl::PremultipliedImage imageDiff{actualImage.size};
+
+        double pixels = 0.0;
+        std::vector<std::string> expectedImagesPaths;
+        for (auto rit = expectations.rbegin(); rit != expectations.rend(); ++rit) {
+            if (mbgl::filesystem::exists(*rit)) {
+                expectedImagesPaths = readExpectedImageEntries(*rit);
+                if (!expectedImagesPaths.empty()) break;
+            }
+        }
+
+        if (expectedImagesPaths.empty()) {
+            metadata.errorMessage = "Failed to find expectations for: " + metadata.paths.stylePath.string();
+            metadata.renderErrored++;
+            return;
+        }
+
+        for (const auto& entry : expectedImagesPaths) {
+            mbgl::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry);
+            if (!maybeExpectedImage) {
+                metadata.errorMessage = "Failed to load expected image " + entry;
+                metadata.renderErrored++;
+                return;
+            }
+
+            metadata.expected = *maybeExpectedImage;
+
+            expectedImage = mbgl::decodeImage(*maybeExpectedImage);
+
+            pixels = // implicitly converting from uint64_t
+                mapbox::pixelmatch(actualImage.data.get(),
+                                   expectedImage.data.get(),
+                                   expectedImage.size.width,
+                                   expectedImage.size.height,
+                                   imageDiff.data.get(),
+                                   0.1285); // Defined in GL JS
+
+            metadata.diff = mbgl::encodePNG(imageDiff);
+
+            mbgl::util::write_file(base + "/diff.png", metadata.diff);
+
+            metadata.difference = pixels / expectedImage.size.area();
+            if (metadata.difference <= metadata.allowed) {
+                break;
+            }
+        }
+
+        if (metadata.difference > metadata.allowed) {
+            metadata.renderFailed++;
+        }
+    }
+}
+
+void TestRunner::checkProbingResults(TestMetadata& resultMetadata) {
+    if (resultMetadata.metrics.isEmpty()) return;
+    const auto writeMetrics = [&resultMetadata](const mbgl::filesystem::path& path,
+                                                const std::string& message = std::string()) {
+        mbgl::filesystem::create_directories(path);
+        mbgl::util::write_file(path / "metrics.json", serializeMetrics(resultMetadata.metrics));
+        resultMetadata.errorMessage += message;
+    };
+
+    const std::vector<mbgl::filesystem::path>& expectedMetrics = resultMetadata.paths.expectedMetrics;
+    if (updateResults == UpdateResults::METRICS) {
+        writeMetrics(expectedMetrics.back(), " Updated expected metrics.");
+        resultMetadata.metricsErrored++;
+        return;
+    }
+
+    // Check the possible paths in reverse order, so that the default path with the test style will only be checked in
+    // the very end.
+    std::vector<std::string> expectedMetricsPaths;
+    for (auto rit = expectedMetrics.rbegin(); rit != expectedMetrics.rend(); ++rit) {
+        if (mbgl::filesystem::exists(*rit)) {
+            expectedMetricsPaths = readExpectedMetricEntries(*rit);
+            if (!expectedMetricsPaths.empty()) break;
+        }
+    }
+
+    // In case no metrics.json is found, skip assigning the expectedMetrics to metadata, otherwise, take the first found
+    // metrics.
+    for (const auto& entry : expectedMetricsPaths) {
+        auto maybeExpectedMetrics = readExpectedMetrics(entry);
+        if (maybeExpectedMetrics.isEmpty()) {
+            resultMetadata.errorMessage = "Failed to load expected metrics " + entry;
+            resultMetadata.metricsErrored++;
+            return;
+        }
+        resultMetadata.expectedMetrics = maybeExpectedMetrics;
+        break;
+    }
+
+    if (resultMetadata.expectedMetrics.isEmpty()) {
+        resultMetadata.errorMessage =
+            "Failed to find metric expectations for: " + resultMetadata.paths.stylePath.string();
+        if (updateResults == UpdateResults::REBASELINE && !resultMetadata.ignoredTest) {
+            writeMetrics(expectedMetrics.back(), ". Created baseline for missing metrics.");
+        }
+        resultMetadata.metricsErrored++;
+        return;
+    }
+
+    // Check file size metrics.
+    auto checkFileSize = [](TestMetadata& metadata) {
+        if (metadata.metrics.fileSize.empty()) return;
+        for (const auto& expected : metadata.expectedMetrics.fileSize) {
+            auto actual = metadata.metrics.fileSize.find(expected.first);
+            if (actual == metadata.metrics.fileSize.end()) {
+                metadata.errorMessage = "Failed to find fileSize probe: " + expected.first;
+                metadata.metricsErrored++;
+                return;
+            }
+            if (actual->second.path != expected.second.path) {
+                std::stringstream ss;
+                ss << "Comparing different files at probe \"" << expected.first << "\": " << actual->second.path
+                   << ", expected is " << expected.second.path << ".";
+                metadata.errorMessage = ss.str();
+                metadata.metricsErrored++;
+                return;
+            }
+
+            auto result = checkValue(expected.second.size, actual->second.size, actual->second.tolerance);
+            if (!std::get<bool>(result)) {
+                std::stringstream ss;
+                ss << "File size does not match at probe \"" << expected.first << "\" for file \""
+                   << expected.second.path << "\": " << actual->second.size << ", expected is " << expected.second.size
+                   << ".";
+
+                metadata.errorMessage += metadata.errorMessage.empty() ? ss.str() : "\n" + ss.str();
+                metadata.metricsFailed++;
+            }
+        }
+    };
+    auto checkMemory = [](TestMetadata& metadata) {
+        if (metadata.metrics.memory.empty()) return;
+#if !defined(SANITIZE)
+        // Check memory metrics.
+        for (const auto& expected : metadata.expectedMetrics.memory) {
+            auto actual = metadata.metrics.memory.find(expected.first);
+            if (actual == metadata.metrics.memory.end()) {
+                metadata.errorMessage = "Failed to find memory probe: " + expected.first;
+                metadata.metricsErrored++;
+                return;
+            }
+            bool passed{false};
+            float delta{0.0f};
+            std::stringstream errorStream;
+            std::tie(passed, delta) = MemoryProbe::checkPeak(expected.second, actual->second);
+            if (!passed) {
+                errorStream << "Allocated memory peak size at probe \"" << expected.first << "\" is "
+                            << actual->second.peak << " bytes, expected is " << expected.second.peak << "±" << delta
+                            << " bytes.";
+
+                metadata.metricsFailed++;
+            }
+
+            std::tie(passed, delta) = MemoryProbe::checkAllocations(expected.second, actual->second);
+            if (!passed) {
+                errorStream << "Number of allocations at probe \"" << expected.first << "\" is "
+                            << actual->second.allocations << ", expected is " << expected.second.allocations << "±"
+                            << std::round(delta) << " allocations.";
+
+                metadata.metricsFailed++;
+            }
+
+            metadata.errorMessage += metadata.errorMessage.empty() ? errorStream.str() : "\n" + errorStream.str();
+        }
+#endif // !defined(SANITIZE)
+    };
+
+    // Check network metrics.
+    auto checkNetwork = [](TestMetadata& metadata) {
+        if (metadata.metrics.network.empty()) return;
+#if !defined(SANITIZE)
+        for (const auto& expected : metadata.expectedMetrics.network) {
+            auto actual = metadata.metrics.network.find(expected.first);
+            if (actual == metadata.metrics.network.end()) {
+                metadata.errorMessage = "Failed to find network probe: " + expected.first;
+                metadata.metricsErrored++;
+                return;
+            }
+            std::stringstream ss;
+            if (actual->second.requests != expected.second.requests) {
+                ss << "Number of requests at probe \"" << expected.first << "\" is " << actual->second.requests
+                   << ", expected is " << expected.second.requests << ". ";
+                metadata.metricsFailed++;
+            }
+            if (actual->second.transferred != expected.second.transferred) {
+                ss << "Transferred data at probe \"" << expected.first << "\" is " << actual->second.transferred
+                   << " bytes, expected is " << expected.second.transferred << " bytes.";
+                metadata.metricsFailed++;
+            }
+            metadata.errorMessage += metadata.errorMessage.empty() ? ss.str() : "\n" + ss.str();
+        }
+#endif // !defined(SANITIZE)
+    };
+    // Check fps metrics
+    auto checkFps = [](TestMetadata& metadata) {
+        if (metadata.metrics.fps.empty()) return;
+        for (const auto& expected : metadata.expectedMetrics.fps) {
+            auto actual = metadata.metrics.fps.find(expected.first);
+            if (actual == metadata.metrics.fps.end()) {
+                metadata.errorMessage = "Failed to find fps probe: " + expected.first;
+                metadata.metricsErrored++;
+                return;
+            }
+            auto result = checkValue(expected.second.average, actual->second.average, expected.second.tolerance);
+            std::stringstream ss;
+            if (!std::get<bool>(result)) {
+                ss << "Average fps at probe \"" << expected.first << "\" is " << actual->second.average
+                   << ", expected to be " << expected.second.average << " with tolerance of "
+                   << expected.second.tolerance;
+                metadata.metricsFailed++;
+            }
+            result = checkValue(expected.second.minOnePc, actual->second.minOnePc, expected.second.tolerance);
+            if (!std::get<bool>(result)) {
+                ss << "Minimum(1%) fps at probe \"" << expected.first << "\" is " << actual->second.minOnePc
+                   << ", expected to be " << expected.second.minOnePc << " with tolerance of "
+                   << expected.second.tolerance;
+                metadata.metricsFailed++;
+            }
+            metadata.errorMessage += metadata.errorMessage.empty() ? ss.str() : "\n" + ss.str();
+        }
+    };
+    // Check gfx metrics
+    auto checkGfx = [](TestMetadata& metadata) {
+        if (metadata.metrics.gfx.empty()) return;
+        for (const auto& expected : metadata.expectedMetrics.gfx) {
+            auto actual = metadata.metrics.gfx.find(expected.first);
+            if (actual == metadata.metrics.gfx.end()) {
+                metadata.errorMessage = "Failed to find gfx probe: " + expected.first;
+                metadata.metricsErrored++;
+                return;
+            }
+
+            const auto& probeName = expected.first;
+            const auto& expectedValue = expected.second;
+            const auto& actualValue = actual->second;
+            std::stringstream ss;
+
+            if (expectedValue.numDrawCalls != actualValue.numDrawCalls) {
+                if (!metadata.errorMessage.empty()) ss << std::endl;
+                ss << "Number of draw calls at probe\"" << probeName << "\" is " << actualValue.numDrawCalls
+                   << ", expected is " << expectedValue.numDrawCalls;
+                metadata.metricsFailed++;
+            }
+
+            if (expectedValue.numTextures != actualValue.numTextures) {
+                if (!metadata.errorMessage.empty()) ss << std::endl;
+                ss << "Number of textures at probe \"" << probeName << "\" is " << actualValue.numTextures
+                   << ", expected is " << expectedValue.numTextures;
+                metadata.metricsFailed++;
+            }
+
+            if (expectedValue.numBuffers != actualValue.numBuffers) {
+                if (!metadata.errorMessage.empty()) ss << std::endl;
+                ss << "Number of vertex and index buffers at probe \"" << probeName << "\" is "
+                   << actualValue.numBuffers << ", expected is " << expectedValue.numBuffers;
+                metadata.metricsFailed++;
+            }
+
+            if (expectedValue.numFrameBuffers != actualValue.numFrameBuffers) {
+                if (!metadata.errorMessage.empty()) ss << std::endl;
+                ss << "Number of frame buffers at probe \"" << probeName << "\" is " << actualValue.numFrameBuffers
+                   << ", expected is " << expectedValue.numFrameBuffers;
+                metadata.metricsFailed++;
+            }
+
+            if (expectedValue.memTextures.peak != actualValue.memTextures.peak) {
+                if (!metadata.errorMessage.empty()) ss << std::endl;
+                ss << "Allocated texture memory peak size at probe \"" << probeName << "\" is "
+                   << actualValue.memTextures.peak << " bytes, expected is " << expectedValue.memTextures.peak
+                   << " bytes";
+                metadata.metricsFailed++;
+            }
+
+            if (expectedValue.memIndexBuffers.peak != actualValue.memIndexBuffers.peak) {
+                if (!metadata.errorMessage.empty()) ss << std::endl;
+                ss << "Allocated index buffer memory peak size at probe \"" << probeName << "\" is "
+                   << actualValue.memIndexBuffers.peak << " bytes, expected is " << expectedValue.memIndexBuffers.peak
+                   << " bytes";
+                metadata.metricsFailed++;
+            }
+
+            if (expectedValue.memVertexBuffers.peak != actualValue.memVertexBuffers.peak) {
+                if (!metadata.errorMessage.empty()) ss << std::endl;
+                ss << "Allocated vertex buffer memory peak size at probe \"" << probeName << "\" is "
+                   << actualValue.memVertexBuffers.peak << " bytes, expected is " << expectedValue.memVertexBuffers.peak
+                   << " bytes";
+                metadata.metricsFailed++;
+            }
+
+            metadata.errorMessage += metadata.errorMessage.empty() ? ss.str() : "\n" + ss.str();
+        }
+    };
+
+    checkFileSize(resultMetadata);
+    checkMemory(resultMetadata);
+    checkNetwork(resultMetadata);
+    checkFps(resultMetadata);
+    checkGfx(resultMetadata);
+
+    if (resultMetadata.ignoredTest) {
+        return;
+    }
+
+    if ((resultMetadata.metricsErrored || resultMetadata.metricsFailed) && updateResults == UpdateResults::REBASELINE) {
+        writeMetrics(expectedMetrics.back(), " Rebaselined expected metric for failed test.");
+    }
+}
+
+namespace {
+
+TestOperation unsupportedOperation(const std::string& operation) {
+    return [operation](TestContext& ctx) {
+        ctx.getMetadata().errorMessage = std::string("Unsupported operation: ") + operation;
+        return false;
+    };
+}
+
+TestOperations getBeforeOperations(const Manifest& manifest) {
+    static const std::string mark = " - default - start";
+    TestOperations result;
+    for (const std::string& probe : manifest.getProbes()) {
+        if (memoryProbeOp == probe) {
+            result.emplace_back([](TestContext& ctx) {
+                assert(!AllocationIndex::isActive());
+                AllocationIndex::setActive(true);
+                ctx.getMetadata().metrics.memory.emplace(std::piecewise_construct,
+                                                         std::forward_as_tuple(memoryProbeOp + mark),
+                                                         std::forward_as_tuple(AllocationIndex::getAllocatedSizePeak(),
+                                                                               AllocationIndex::getAllocationsCount()));
+                return true;
+            });
+            continue;
+        }
+        if (gfxProbeOp == probe) {
+            result.emplace_back([](TestContext& ctx) {
+                assert(!ctx.gfxProbeActive);
+                ctx.gfxProbeActive = true;
+                ctx.baselineGfxProbe = ctx.activeGfxProbe;
+                return true;
+            });
+            continue;
+        }
+
+        if (networkProbeOp == probe) {
+            result.emplace_back([](TestContext& ctx) {
+                assert(!ProxyFileSource::isTrackingActive());
+                ProxyFileSource::setTrackingActive(true);
+                ctx.getMetadata().metrics.network.emplace(
+                    std::piecewise_construct,
+                    std::forward_as_tuple(networkProbeOp + mark),
+                    std::forward_as_tuple(ProxyFileSource::getRequestCount(), ProxyFileSource::getTransferredSize()));
+                return true;
+            });
+            continue;
+        }
+        result.emplace_back(unsupportedOperation(probe));
+    }
+    return result;
+}
+
+TestOperations getAfterOperations(const Manifest& manifest) {
+    static const std::string mark = " - default - end";
+    TestOperations result;
+    for (const std::string& probe : manifest.getProbes()) {
+        if (memoryProbeOp == probe) {
+            result.emplace_back([](TestContext& ctx) {
+                assert(AllocationIndex::isActive());
+                auto emplaced = ctx.getMetadata().metrics.memory.emplace(
+                    std::piecewise_construct,
+                    std::forward_as_tuple(memoryProbeOp + mark),
+                    std::forward_as_tuple(AllocationIndex::getAllocatedSizePeak(),
+                                          AllocationIndex::getAllocationsCount()));
+                assert(emplaced.second);
+                // TODO: Improve tolerance handling for memory tests.
+                emplaced.first->second.tolerance = 0.2f;
+                AllocationIndex::setActive(false);
+                AllocationIndex::reset();
+                return true;
+            });
+            continue;
+        }
+        if (gfxProbeOp == probe) {
+            result.emplace_back([](TestContext& ctx) {
+                // Compare memory allocations to the baseline probe
+                GfxProbe metricProbe = ctx.activeGfxProbe;
+                metricProbe.memIndexBuffers.peak -= ctx.baselineGfxProbe.memIndexBuffers.peak;
+                metricProbe.memVertexBuffers.peak -= ctx.baselineGfxProbe.memVertexBuffers.peak;
+                metricProbe.memTextures.peak -= ctx.baselineGfxProbe.memTextures.peak;
+                ctx.getMetadata().metrics.gfx.insert({gfxProbeOp + mark, metricProbe});
+
+                ctx.gfxProbeActive = false;
+                return true;
+            });
+            continue;
+        }
+        if (networkProbeOp == probe) {
+            result.emplace_back([](TestContext& ctx) {
+                assert(ProxyFileSource::isTrackingActive());
+                ctx.getMetadata().metrics.network.emplace(
+                    std::piecewise_construct,
+                    std::forward_as_tuple(networkProbeOp + mark),
+                    std::forward_as_tuple(ProxyFileSource::getRequestCount(), ProxyFileSource::getTransferredSize()));
+                ProxyFileSource::setTrackingActive(false);
+                return true;
+            });
+            continue;
+        }
+        result.emplace_back(unsupportedOperation(probe));
+    }
+    return result;
+}
+
+void resetContext(const TestMetadata& metadata, TestContext& ctx) {
+    ctx.getFrontend().getRenderer()->clearData();
+    ctx.getFrontend().setSize(metadata.size);
+    auto& map = ctx.getMap();
+    map.setSize(metadata.size);
+    map.setProjectionMode(mbgl::ProjectionMode()
+                              .withAxonometric(metadata.axonometric)
+                              .withXSkew(metadata.xSkew)
+                              .withYSkew(metadata.ySkew));
+    map.setDebug(metadata.debug);
+    map.getStyle().loadJSON(serializeJsonValue(metadata.document));
+}
+
+LatLng getTileCenterCoordinates(const UnwrappedTileID& tileId) {
+    double scale = (1 << tileId.canonical.z);
+    Point<double> tileCenter{(tileId.canonical.x + 0.5) * util::tileSize, (tileId.canonical.y + 0.5) * util::tileSize};
+    return Projection::unproject(tileCenter, scale);
+}
+
+uint32_t getTileScreenPixelSize(float pixelRatio) {
+    return util::tileSize * pixelRatio;
+}
+
+uint32_t getImageTileOffset(const std::set<uint32_t>& dims, uint32_t dim, float pixelRatio) {
+    auto it = dims.find(dim);
+    if (it == dims.end()) {
+        assert(false);
+        return 0;
+    }
+    return static_cast<uint32_t>(std::distance(dims.begin(), it)) * getTileScreenPixelSize(pixelRatio);
+}
+
+} // namespace
+
+TestRunner::Impl::Impl(const TestMetadata& metadata, const mbgl::ResourceOptions& resourceOptions)
+    : observer(std::make_unique<TestRunnerMapObserver>()),
+      frontend(metadata.size, metadata.pixelRatio, swapBehavior(metadata.mapMode)),
+      fileSource(mbgl::FileSourceManager::get()->getFileSource(mbgl::FileSourceType::ResourceLoader, resourceOptions)),
+      map(frontend,
+          *observer.get(),
+          mbgl::MapOptions()
+              .withMapMode(metadata.mapMode)
+              .withSize(metadata.size)
+              .withPixelRatio(metadata.pixelRatio)
+              .withCrossSourceCollisions(metadata.crossSourceCollisions),
+          resourceOptions) {}
+
+TestRunner::Impl::~Impl() {}
+
+void TestRunner::run(TestMetadata& metadata) {
+    AllocationIndex::setActive(false);
+    AllocationIndex::reset();
+    ProxyFileSource::setTrackingActive(false);
+
+    struct ContextImpl final : public TestContext {
+        ContextImpl(TestMetadata& metadata_) : metadata(metadata_) {}
+        HeadlessFrontend& getFrontend() override {
+            assert(runnerImpl);
+            return runnerImpl->frontend;
+        }
+        Map& getMap() override {
+            assert(runnerImpl);
+            return runnerImpl->map;
+        }
+
+        FileSource& getFileSource() override {
+            assert(runnerImpl);
+            return *runnerImpl->fileSource;
+        }
+
+        TestRunnerMapObserver& getObserver() override {
+            assert(runnerImpl);
+            return *runnerImpl->observer;
+        }
+        TestMetadata& getMetadata() override { return metadata; }
+
+        TestRunner::Impl* runnerImpl = nullptr;
+        TestMetadata& metadata;
+    };
+
+    ContextImpl ctx(metadata);
+
+    if (!metadata.ignoredTest) {
+        for (const auto& operation : getBeforeOperations(manifest)) {
+            if (!operation(ctx)) return;
+        }
+    }
+
+    std::string key = mbgl::util::toString(uint32_t(metadata.mapMode)) + "/" +
+                      mbgl::util::toString(metadata.pixelRatio) + "/" +
+                      mbgl::util::toString(uint32_t(metadata.crossSourceCollisions));
+
+    if (maps.find(key) == maps.end()) {
+        maps[key] = std::make_unique<TestRunner::Impl>(
+            metadata,
+            mbgl::ResourceOptions().withCachePath(manifest.getCachePath()).withAccessToken(manifest.getAccessToken()));
+    }
+
+    ctx.runnerImpl = maps[key].get();
+    auto& frontend = ctx.getFrontend();
+    auto& map = ctx.getMap();
+
+    resetContext(metadata, ctx);
+    auto camera = map.getStyle().getDefaultCamera();
+
+    HeadlessFrontend::RenderResult result{};
+    std::string cutOffLabelsReport;
+    std::string duplicationsReport;
+
+    if (metadata.mapMode == MapMode::Tile) {
+        assert(camera.zoom);
+        assert(camera.center);
+        auto tileIds = util::tileCover(map.latLngBoundsForCamera(camera), *camera.zoom);
+        assert(!tileIds.empty());
+        std::set<uint32_t> xDims;
+        std::set<uint32_t> yDims;
+        struct SymbolLocationAndStatus {
+            UnwrappedTileID tileId;
+            mapbox::geometry::box<float> location;
+            bool placed;
+            bool isIcon;
+        };
+        std::map<std::u16string, std::vector<SymbolLocationAndStatus>> symbolIndex;
+
+        for (const auto& tileId : tileIds) {
+            xDims.insert(tileId.canonical.x);
+            yDims.insert(tileId.canonical.y);
+            assert(tileId.canonical.z == uint8_t(*camera.zoom));
+        }
+        auto tileScreenSize = getTileScreenPixelSize(metadata.pixelRatio);
+
+        result.image =
+            PremultipliedImage({uint32_t(xDims.size()) * tileScreenSize, uint32_t(yDims.size()) * tileScreenSize});
+        for (const auto& tileId : tileIds) {
+            resetContext(metadata, ctx);
+            ctx.getFrontend().getRenderer()->collectPlacedSymbolData(true);
+            auto cameraForTile{camera};
+            cameraForTile.withCenter(getTileCenterCoordinates(tileId));
+            map.jumpTo(cameraForTile);
+
+            auto resultForTile = runTest(metadata, ctx);
+            if (!resultForTile.image.valid()) {
+                metadata.errorMessage = "Failed rendering tile: " + util::toString(tileId);
+                return;
+            }
+            auto xOffset = getImageTileOffset(xDims, tileId.canonical.x, metadata.pixelRatio);
+            auto yOffset = getImageTileOffset(yDims, tileId.canonical.y, metadata.pixelRatio);
+
+            const auto& placedSymbols = ctx.getFrontend().getRenderer()->getPlacedSymbolsData();
+
+            auto findCutOffs =
+                [&](const PlacedSymbolData& placedSymbol, mapbox::geometry::box<float> box, bool placed, bool isIcon) {
+                    box.min.x += xOffset - placedSymbol.viewportPadding;
+                    box.max.x += xOffset - placedSymbol.viewportPadding;
+                    box.min.y += yOffset - placedSymbol.viewportPadding;
+                    box.max.y += yOffset - placedSymbol.viewportPadding;
+
+                    auto& symbols = symbolIndex[placedSymbol.key];
+                    auto it = std::find_if(symbols.begin(), symbols.end(), [isIcon, box](const auto& a) {
+                        return a.isIcon == isIcon && a.location.min == box.min && a.location.max == box.max;
+                    });
+                    static std::wstring_convert<std::codecvt_utf8<char16_t>, char16_t> cv;
+                    if (it == symbols.end()) {
+                        symbols.push_back({tileId, box, placed, isIcon});
+                        return;
+                    }
+                    if (tileId == it->tileId && isIcon == it->isIcon) {
+                        std::stringstream ss;
+                        ss << std::endl
+                           << ++metadata.duplicationsCount << ". \"" << cv.to_bytes(placedSymbol.key) << "\" "
+                           << (isIcon ? "icon" : "text") << " ((" << box.min.x << ", " << box.min.y << "), ("
+                           << box.max.x << ", " << box.max.y << ")) from layer: " << placedSymbol.layer << " at "
+                           << util::toString(tileId);
+                        duplicationsReport += ss.str();
+                        return;
+                    }
+                    if (it->placed != placed && isIcon == it->isIcon) {
+                        std::stringstream ss;
+                        ss << std::endl
+                           << ++metadata.labelCutOffFound << ". \"" << cv.to_bytes(placedSymbol.key) << "\" "
+                           << (isIcon ? "icon" : "text") << " ((" << box.min.x << ", " << box.min.y << "), ("
+                           << box.max.x << ", " << box.max.y << ")) from layer: " << placedSymbol.layer
+                           << (placed ? " is placed at " : " is not placed at ") << util::toString(tileId) << " and"
+                           << (it->placed ? " placed at " : " not placed at ") << util::toString(it->tileId);
+                        cutOffLabelsReport += ss.str();
+                    }
+                };
+
+            for (const auto& placedSymbol : placedSymbols) {
+                if (placedSymbol.intersectsTileBorder) {
+                    if (placedSymbol.textCollisionBox) {
+                        findCutOffs(
+                            placedSymbol, *placedSymbol.textCollisionBox, placedSymbol.textPlaced, false /*isIcon*/);
+                    }
+                    if (placedSymbol.iconCollisionBox) {
+                        findCutOffs(
+                            placedSymbol, *placedSymbol.iconCollisionBox, placedSymbol.iconPlaced, true /*isIcon*/);
+                    }
+                }
+            }
+
+            PremultipliedImage::copy(
+                resultForTile.image, result.image, {0, 0}, {xOffset, yOffset}, resultForTile.image.size);
+            result.stats += resultForTile.stats;
+        }
+    } else {
+        map.jumpTo(camera);
+        result = runTest(metadata, ctx);
+    }
+
+    if (!metadata.ignoredTest) {
+        ctx.activeGfxProbe = GfxProbe(result.stats, ctx.activeGfxProbe);
+        for (const auto& operation : getAfterOperations(manifest)) {
+            if (!operation(ctx)) return;
+        }
+    }
+
+    if (metadata.renderTest) {
+        checkProbingResults(metadata);
+        appendLabelCutOffResults(metadata, cutOffLabelsReport, duplicationsReport);
+        checkRenderTestResults(std::move(result.image), metadata);
+    } else {
+        std::vector<mbgl::Feature> features;
+        assert(metadata.document["metadata"]["test"]["queryGeometry"].IsArray());
+        if (metadata.document["metadata"]["test"]["queryGeometry"][0].IsNumber() &&
+            metadata.document["metadata"]["test"]["queryGeometry"][1].IsNumber()) {
+            features = frontend.getRenderer()->queryRenderedFeatures(metadata.queryGeometry, metadata.queryOptions);
+        } else {
+            features = frontend.getRenderer()->queryRenderedFeatures(metadata.queryGeometryBox, metadata.queryOptions);
+        }
+        checkQueryTestResults(std::move(result.image), std::move(features), metadata);
+    }
+}
+
+void TestRunner::appendLabelCutOffResults(TestMetadata& resultMetadata,
+                                          const std::string& cutOffLabelsReport,
+                                          const std::string& duplicationsReport) {
+    if (resultMetadata.labelCutOffFound) { // Append label cut-off statistics.
+        resultMetadata.errorMessage += "\n Label cut-offs:";
+        resultMetadata.errorMessage += cutOffLabelsReport;
+        if (resultMetadata.duplicationsCount) {
+            resultMetadata.errorMessage += "\n\n Label duplications:";
+            resultMetadata.errorMessage += duplicationsReport;
+        }
+    }
+}
+
+mbgl::HeadlessFrontend::RenderResult TestRunner::runTest(TestMetadata& metadata, TestContext& ctx) {
+    HeadlessFrontend::RenderResult result{};
+    for (const auto& operation : parseTestOperations(metadata)) {
+        if (!operation(ctx)) return result;
+    }
+
+    try {
+        if (metadata.outputsImage) result = ctx.getFrontend().render(ctx.getMap());
+    } catch (const std::exception& e) {
+        ctx.getMetadata().errorMessage = std::string("Renering raised an exception: ") + e.what();
+    }
+    return result;
+}
+
+void TestRunner::reset() {
+    maps.clear();
+}

render-test/runner.hpp

@@ -0,0 +1,71 @@
+#pragma once
+
+#include <mbgl/gfx/headless_frontend.hpp>
+#include <mbgl/map/map.hpp>
+#include <mbgl/storage/file_source.hpp>
+
+#include "manifest_parser.hpp"
+
+#include <memory>
+#include <string>
+
+class TestRunnerMapObserver;
+struct TestMetadata;
+
+class TestRunnerMapObserver : public mbgl::MapObserver {
+public:
+    TestRunnerMapObserver() = default;
+    void onDidFailLoadingMap(mbgl::MapLoadError, const std::string&) override { mapLoadFailure = true; }
+
+    void onDidFinishRenderingMap(RenderMode mode) override final {
+        if (!finishRenderingMap) finishRenderingMap = mode == RenderMode::Full;
+    }
+
+    void onDidBecomeIdle() override final { idle = true; }
+
+    void reset() {
+        mapLoadFailure = false;
+        finishRenderingMap = false;
+        idle = false;
+    }
+
+    bool mapLoadFailure;
+    bool finishRenderingMap;
+    bool idle;
+};
+
+class TestRunner {
+public:
+    enum class UpdateResults { NO, DEFAULT, PLATFORM, METRICS, REBASELINE };
+
+    TestRunner(Manifest, UpdateResults);
+    void run(TestMetadata&);
+    void reset();
+
+    // Manifest
+    const Manifest& getManifest() const;
+    void doShuffle(uint32_t seed);
+
+private:
+    mbgl::HeadlessFrontend::RenderResult runTest(TestMetadata& metadata, TestContext& ctx);
+    void checkQueryTestResults(mbgl::PremultipliedImage&& actualImage,
+                               std::vector<mbgl::Feature>&& features,
+                               TestMetadata&);
+    void checkRenderTestResults(mbgl::PremultipliedImage&& image, TestMetadata&);
+    void checkProbingResults(TestMetadata&);
+    void appendLabelCutOffResults(TestMetadata&, const std::string&, const std::string&);
+    void registerProxyFileSource();
+
+    struct Impl {
+        Impl(const TestMetadata&, const mbgl::ResourceOptions&);
+        ~Impl();
+
+        std::unique_ptr<TestRunnerMapObserver> observer;
+        mbgl::HeadlessFrontend frontend;
+        std::shared_ptr<mbgl::FileSource> fileSource;
+        mbgl::Map map;
+    };
+    std::unordered_map<std::string, std::unique_ptr<Impl>> maps;
+    Manifest manifest;
+    UpdateResults updateResults;
+};