Store Android benchmark results on device (#2844)

.github/workflows/android-device-test.yml

@@ -158,14 +158,8 @@ jobs:
           externalData: ${{ env.external_data_arn }}
           testSpecArn: ${{ matrix.test.testSpecArn }}
 
-      - name: Upload benchmark results to S3
+      - name: Store Test Artifacts
-        if: always() && env.run_device_test == 'true' && matrix.test.name == 'Android Benchmark'
+        if: (matrix.test.name == 'Android Benchmark' || failure()) && env.run_device_test == 'true'
-        run: |
-          npm install
-          node scripts/aws-device-farm/download-benchmark-results.mjs ${{ steps.aws_device_farm_run.outputs.runArn }}
-
-      - name: Log Test Spec Output
-        if: failure() && env.run_device_test == 'true'
         run: |
           npm install
           temp_dir="$(mktemp -d)"
@@ -173,7 +167,7 @@ jobs:
           zip -r test_artifacts.zip "$temp_dir"
 
       - name: Upload Test Artifacts
-        if: failure() && env.run_device_test == 'true'
+        if: (matrix.test.name == 'Android Benchmark' || failure()) && env.run_device_test == 'true'
         uses: actions/upload-artifact@v4
         with:
           name: "Test Artifacts ${{ matrix.test.name }}"

platform/android/MapLibreAndroidTestApp/src/main/AndroidManifest.xml

@@ -13,7 +13,9 @@
         android:label="@string/app_name"
         android:roundIcon="@drawable/ic_launcher_round"
         android:supportsRtl="true"
-        android:theme="@style/AppTheme">
+        android:theme="@style/AppTheme"
+        android:debuggable="true"
+        tools:ignore="HardcodedDebugMode">
         <activity
             android:name=".activity.FeatureOverviewActivity"
             android:exported="true"

platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/activity/benchmark/BenchmarkActivity.kt

@@ -18,10 +18,6 @@ import kotlinx.serialization.json.jsonArray
 import kotlinx.serialization.json.jsonObject
 import kotlinx.serialization.json.jsonPrimitive
 import kotlinx.serialization.json.putJsonObject
-import okhttp3.MediaType.Companion.toMediaType
-import okhttp3.OkHttpClient
-import okhttp3.Request
-import okhttp3.RequestBody.Companion.toRequestBody
 import org.maplibre.android.BuildConfig.GIT_REVISION
 import org.maplibre.android.camera.CameraUpdateFactory
 import org.maplibre.android.geometry.LatLng
@@ -39,7 +35,6 @@ import java.io.File
 data class BenchmarkInputData(
     val styleNames: List<String>,
     val styleURLs: List<String>,
-    val resultsAPI: String = ""
 ) {
     init {
         if (styleNames.size != styleURLs.size)
@@ -47,10 +42,6 @@ data class BenchmarkInputData(
     }
 }
 
-/**
- * Prepares JSON payload that is sent to the API that collects benchmark results.
- * See https://github.com/maplibre/ci-runners
- */
 @SuppressLint("NewApi")
 fun jsonPayload(styleNames: List<String>, fpsResults: BenchmarkResults, encodingTimeResults: BenchmarkResults, renderingTimeResults: BenchmarkResults): JsonObject {
     return buildJsonObject {
@@ -141,14 +132,12 @@ class BenchmarkActivity : AppCompatActivity() {
             val jsonElement = Json.parseToJsonElement(jsonFileContents)
             val styleNames = jsonElement.jsonObject["styleNames"]?.jsonArray?.map { it.jsonPrimitive.content }
             val styleURLs = jsonElement.jsonObject["styleURLs"]?.jsonArray?.map { it.jsonPrimitive.content }
-            val resultsAPI = jsonElement.jsonObject["resultsAPI"]?.jsonPrimitive?.content
+            if (styleNames == null || styleURLs == null) {
-            if (styleNames == null || styleURLs == null || resultsAPI == null) {
                 throw Error("${jsonFile.name} is missing elements")
             }
             return BenchmarkInputData(
                 styleNames = styleNames.toList(),
-                styleURLs = styleURLs.toList(),
+                styleURLs = styleURLs.toList()
-                resultsAPI = resultsAPI
             )
         } else {
             Logger.i(TAG, "${jsonFile.name} not found, reading from developer-config.xml")
@@ -166,8 +155,20 @@ class BenchmarkActivity : AppCompatActivity() {
 
         // return default
         return BenchmarkInputData(
-            styleNames = listOf("MapLibre Demotiles"),
+            styleNames = listOf(
-            styleURLs = listOf("https://demotiles.maplibre.org/style.json")
+                "AWS Open Data Standard Light",
+                "Facebook Light",
+                "Americana",
+                "Protomaps Light",
+                "Versatiles Colorful"
+            ),
+            styleURLs = listOf(
+                "https://maps.geo.us-east-2.amazonaws.com/maps/v0/maps/OpenDataStyle/style-descriptor?key=v1.public.eyJqdGkiOiI1NjY5ZTU4My0yNWQwLTQ5MjctODhkMS03OGUxOTY4Y2RhMzgifR_7GLT66TNRXhZJ4KyJ-GK1TPYD9DaWuc5o6YyVmlikVwMaLvEs_iqkCIydspe_vjmgUVsIQstkGoInXV_nd5CcmqRMMa-_wb66SxDdbeRDvmmkpy2Ow_LX9GJDgL2bbiCws0wupJPFDwWCWFLwpK9ICmzGvNcrPbX5uczOQL0N8V9iUvziA52a1WWkZucIf6MUViFRf3XoFkyAT15Ll0NDypAzY63Bnj8_zS8bOaCvJaQqcXM9lrbTusy8Ftq8cEbbK5aMFapXRjug7qcrzUiQ5sr0g23qdMvnKJQFfo7JuQn8vwAksxrQm6A0ByceEXSfyaBoVpFcTzEclxUomhY.NjAyMWJkZWUtMGMyOS00NmRkLThjZTMtODEyOTkzZTUyMTBi",
+                "https://external.xx.fbcdn.net/maps/vt/style/canterbury_1_0/?locale=en_US",
+                "https://americanamap.org/style.json",
+                "https://api.protomaps.com/styles/v2/light.json?key=e761cc7daedf832a",
+                "https://tiles.versatiles.org/assets/styles/colorful.json",
+            )
         )
     }
 
@@ -313,28 +314,16 @@ class BenchmarkActivity : AppCompatActivity() {
         mapView.onLowMemory()
     }
 
-    private fun sendResults() {
+    private fun storeResults() {
-        val api = inputData.resultsAPI
-        if (api.isEmpty()) {
-            Logger.i(TAG, "Not sending results to API")
-            return
-        }
-
-        val client = OkHttpClient()
-
         val payload = jsonPayload(inputData.styleNames, fpsResults, encodingTimeResults, renderingTimeResults)
-        Logger.i(TAG, "Sending JSON payload to API: $payload")
 
-        val request = Request.Builder()
+        val dataDir = this.filesDir
-            .url(api)
+        val benchmarkResultsFile = File(dataDir, "benchmark_results.json")
-            .post(
+        benchmarkResultsFile.writeText(Json.encodeToString(payload))
-                Json.encodeToString(payload).toRequestBody("application/json".toMediaType()))
-            .build()
-        client.newCall(request).execute()
     }
 
     private fun benchmarkDone() {
-        sendResults()
+        storeResults()
         setResult(Activity.RESULT_OK)
         finish()
     }

platform/android/gradle.properties

@@ -3,4 +3,5 @@ systemProp.org.gradle.internal.http.connectionTimeout=360000
 systemProp.org.gradle.internal.http.socketTimeout=360000
 org.gradle.jvmargs=-Xmx4096M
 android.nonTransitiveRClass=false
-android.nonFinalResIds=false
+android.nonFinalResIds=false
+android.injected.androidTest.leaveApksInstalledAfterRun=true

platform/android/settings.gradle

@@ -8,10 +8,10 @@ rootProject.name = "MapLibre Native for Android"
 
 def renderTestProjectDir = new File(rootDir, '../../render-test/android')
 includeBuild(renderTestProjectDir) {
-    name ="Render Test App"
+    name ="renderTestApp"
 }
 
 def cppTestProjectDir = new File(rootDir, '../../test/android')
 includeBuild(cppTestProjectDir) {
-    name ="C++ Unit Test App"
+    name ="cppUnitTestsApp"
 }

scripts/aws-device-farm/download-benchmark-results.mjs

@@ -1,129 +0,0 @@
-// This script parses logcat output from AWS Device Farm to extract benchmark results.
-// It also uploads the results to S3.
-// android-benchmark-render/{gitRevision}/{deviceModel}.json
-
-import https from "node:https";
-import readline from "node:readline";
-
-import {
-  ListJobsCommand,
-  ListArtifactsCommand,
-} from "@aws-sdk/client-device-farm";
-import { PutObjectCommand } from "@aws-sdk/client-s3";
-import { getDeviceFarmClient } from "./device-farm-client.mjs";
-import { getS3Client } from "./s3-client.mjs";
-
-/**
- * Retrieves a log file and looks for the line that includes the benchmark results.
- * Resolves to the log line or an empty string if the log line was not found.
- * 
- * @param {string} artifactUrl
- * @returns {Promise<string>}
- */
-function getBenchmarkResult(artifactUrl) {
-  return new Promise((resolve, reject) => {
-    https.get(artifactUrl, (res) => {
-      const rl = readline.createInterface({
-        input: res,
-      });
-
-      rl.on("line", (line) => {
-        if (line.includes('Benchmark {"resultsPerStyle"')) {
-          resolve(line);
-          rl.close();
-        }
-      });
-
-      rl.on("close", () => {
-        resolve("");
-      });
-    });
-  });
-}
-
-function usage() {
-  console.error("Downloads benchmark results from AWS Device Farm");
-  console.error(`Usage: node ${process.argv.at(1)} RUN_ARN`);
-  process.exit(1);
-}
-
-const deviceFarmClient = getDeviceFarmClient();
-
-if (process.argv.length !== 3) usage();
-
-const arn = process.argv.at(2);
-
-const listJobsCommand = new ListJobsCommand({ arn });
-const { jobs } = await deviceFarmClient.send(listJobsCommand);
-
-if (!jobs) throw new Error("Failed to retrieve jobs");
-
-const passedJobs = jobs.filter((job) => job.result === "PASSED" && job.arn);
-const failedJobs = jobs.filter((job) => job.result !== "PASSED" || !job.arn);
-
-failedJobs.forEach((job) => {
-  console.error(
-    `job.result not PASSED. Skipping.\n$${JSON.stringify(job, null, 0)}`
-  );
-});
-
-// retrieve all artifacts that are files (logcat logs are file artifacts)
-const artifacts = await Promise.all(
-  passedJobs.map((job) =>
-    deviceFarmClient.send(
-      new ListArtifactsCommand({
-        arn: job.arn,
-        type: "FILE",
-      })
-    )
-  )
-);
-
-// only consider artifacts that are logcat logs
-const logcatArtifacts = artifacts.flatMap((listArtifactsOutput) =>
-  listArtifactsOutput.artifacts
-    ? listArtifactsOutput.artifacts.filter(
-        (artifact) =>
-          artifact.type === "DEVICE_LOG" && artifact.name === "Logcat"
-      )
-    : []
-);
-
-const benchmarkResultLogLines = await Promise.all(
-  logcatArtifacts.flatMap((artifact) => artifact.url ? [getBenchmarkResult(artifact.url)] : [])
-);
-
-// parse the log lines to objects
-/** @type {{gitRevision?: string, model?: string}[]} */
-const benchmarkResults = benchmarkResultLogLines
-  .map((line) => {
-    const firstCurly = line.indexOf("{");
-    if (firstCurly === -1) return "";
-    return line.substring(firstCurly);
-  })
-  .filter((line) => !!line)
-  .map((jsonVal) => JSON.parse(jsonVal));
-
-const s3Client = getS3Client();
-
-// store benchmark results as JSON files on S3
-await Promise.all(
-  benchmarkResults.flatMap((result) => {
-    if (typeof result.model !== "string") {
-      console.error("Result missing model");
-      return [];
-    }
-    if (typeof result.gitRevision !== "string") {
-      console.error("Result missing gitRevision");
-      return [];
-    }
-
-    const command = new PutObjectCommand({
-      Bucket: "maplibre-native", // use environment variable
-      Key: `android-benchmark-render/${result.gitRevision}/${result.model}.json`,
-      ContentType: "application/json",
-      Body: JSON.stringify(result, null, 2),
-    });
-    return [s3Client.send(command)];
-  })
-);