Run Android Instrumentation Tests as part of PR workflow (#2309)

.github/actions/aws-device-farm-run/action.yml

@@ -121,6 +121,7 @@ runs:
             --device-pool-arn ${{ inputs.AWS_DEVICE_FARM_DEVICE_POOL_ARN }} \
             --test type=${{ inputs.testType }},testPackageArn=${{ env.test_package_arn }}${{ inputs.testFilter && ',filter=' }}${{ inputs.testFilter }}${{ inputs.testSpecArn && ',testSpecArn=' }}${{ inputs.testSpecArn }} \
             ${{ inputs.externalData && '--configuration extraDataPackageArn=' }}${{ inputs.externalData }} \
+            --execution-configuration videoCapture=false \
             --output text --query "run.arn")"
           
           echo "runArn=$arn" >> "$GITHUB_OUTPUT"

.github/workflows/android-ci.yml

@@ -137,72 +137,20 @@ jobs:
       - if: github.event_name == 'pull_request'
         uses: ./.github/actions/save-pr-number
 
-      - name: Build UI tests
-        if: github.ref == 'refs/heads/main'
+      - name: Build Instrumentation Tests, copy to platform/android
         run: |
           ./gradlew assembleLegacyDebug assembleLegacyDebugAndroidTest -PtestBuildType=debug
+          cp MapLibreAndroidTestApp/build/outputs/apk/legacy/debug/MapLibreAndroidTestApp-legacy-debug.apk InstrumentationTestApp.apk
+          cp MapLibreAndroidTestApp/build/outputs/apk/androidTest/legacy/debug/MapLibreAndroidTestApp-legacy-debug-androidTest.apk InstrumentationTests.apk
 
-      - name: Configure AWS Credentials
-        if: github.ref == 'refs/heads/main'
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-west-2
-          role-to-assume: ${{ vars.AWS_ROLE_TO_ASSUME }}
-          role-duration-seconds: 21600
-          role-session-name: MySessionName
-
-      - name: Create upload
-        if: github.ref == 'refs/heads/main'
-        uses: realm/aws-devicefarm/create-upload@master
-        id: upload-android-app
-        with:
-          project_arn: ${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }}
-          name: MapLibreAndroidTestApp-debug.apk
-          type: ANDROID_APP
-
-      - name: Create upload
-        if: github.ref == 'refs/heads/main'
-        uses: realm/aws-devicefarm/create-upload@master
-        id: upload-android-test
-        with:
-          project_arn: ${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }}
-          name: MapLibreAndroidTestApp-debug-androidTest.apk
-          type: INSTRUMENTATION_TEST_PACKAGE
-
-      - name: Upload Android UI test
-        if: github.ref == 'refs/heads/main'
-        run: |
-          curl -T MapLibreAndroidTestApp/build/outputs/apk/legacy/debug/MapLibreAndroidTestApp-legacy-debug.apk '${{ steps.upload-android-app.outputs.url }}'
-          curl -T MapLibreAndroidTestApp/build/outputs/apk/androidTest/legacy/debug/MapLibreAndroidTestApp-legacy-debug-androidTest.apk '${{ steps.upload-android-test.outputs.url }}'
-
-      - name: Write uploads.env
-        if: github.ref == 'refs/heads/main'
-        working-directory: .
-        run: |
-          echo "ANDROID_APP_ARN=${{ steps.upload-android-app.outputs.arn }}" >> uploads.env
-          echo "ANDROID_TEST_ARN=${{ steps.upload-android-test.outputs.arn }}" >> uploads.env
-
-      - uses: actions/upload-artifact@v4
-        if: github.ref == 'refs/heads/main'
-        with:
-          if-no-files-found: error
-          name: uploadsEnv
-          path: uploads.env
-
-      - name: Store debug artifacts
+      - name: Upload android-ui-test
         uses: actions/upload-artifact@v4
         with:
-          name: debug-artifacts
+          if-no-files-found: error
+          name: android-ui-test
           path: |
-            MapLibreAndroidTestApp/build/outputs/apk/debug
-            MapLibreAndroid/build/reports/lint-results.html
-            MapLibreAndroid/lint-baseline.xml
-            MapLibreAndroidTestApp/build/reports/lint-results.html
-            MapLibreAndroidTestApp/build/reports/lint-results.xml
-            MapLibreAndroidTestApp/lint-baseline.xml
-            MapLibreAndroid/build/intermediates/cmake/debug/obj
+            platform/android/InstrumentationTestApp.apk
+            platform/android/InstrumentationTests.apk
 
   android-build-render-test:
     runs-on: ubuntu-latest
@@ -240,42 +188,6 @@ jobs:
             ./render-test/android/RenderTestsApp.apk
             ./render-test/android/RenderTests.apk
 
-  android-instrumentation-test:
-    needs: android-build
-
-    runs-on: ubuntu-latest
-    if: github.repository_owner == 'maplibre' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/android-annotations')
-    steps:
-      - uses: actions/download-artifact@v4
-        with:
-          name: uploadsEnv
-
-      - name: Read uploads.env
-        run: cat uploads.env >> "$GITHUB_ENV"
-
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-west-2
-          role-to-assume: ${{ vars.AWS_ROLE_TO_ASSUME }}
-          role-duration-seconds: 21600
-          role-session-name: MySessionName
-
-      - name: Schedule test run
-        uses: realm/aws-devicefarm/test-application@master
-        with:
-          name: MapLibre Native Android Instrumentation Test
-          project_arn: ${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }}
-          device_pool_arn: ${{ vars.AWS_DEVICE_FARM_DEVICE_POOL_ARN }}
-          app_arn: ${{ env.ANDROID_APP_ARN }}
-          test_spec_arn: ${{ vars.AWS_DEVICE_FARM_TEST_SPEC_ARN_ANDROID_UI_TESTS }}
-          app_type: ANDROID_APP
-          test_type: INSTRUMENTATION
-          test_package_arn: ${{ env.ANDROID_TEST_ARN }}
-          timeout: 28800
-
   android-ci-result:
     runs-on: ubuntu-latest
     if: needs.pre_job.outputs.should_skip != 'true' && always()

.github/workflows/android-device-test.yml

@@ -10,7 +10,6 @@ jobs:
   android-device-test:
     strategy:
       fail-fast: false
-      max-parallel: 2
       matrix:
         test: [
           {
@@ -32,6 +31,15 @@ jobs:
             # benchmark-android.yaml
             # see https://github.com/maplibre/ci-runners/tree/main/aws-device-farm/custom-test-envs
             testSpecArn: "arn:aws:devicefarm:us-west-2:373521797162:upload:20687d72-0e46-403e-8f03-0941850665bc/14862afb-cf88-44aa-9f1e-5131cbb22f01"
+          },
+          {
+            artifactName: android-ui-test,
+            testFile: InstrumentationTests.apk,
+            appFile: InstrumentationTestApp.apk,
+            name: "Android Instrumentation Tests",
+            # Google Pixel 7 Pro
+            devicePool: "arn:aws:devicefarm:us-west-2:373521797162:devicepool:20687d72-0e46-403e-8f03-0941850665bc/9692fe7f-86a9-4ecc-908f-175600968564",
+            testSpecArn: "arn:aws:devicefarm:us-west-2:373521797162:upload:20687d72-0e46-403e-8f03-0941850665bc/09e0738e-c91e-4c5f-81e6-06a06cc340d8"
           }
         ]
     runs-on: ubuntu-latest
@@ -76,7 +84,7 @@ jobs:
           echo "run_device_test=true" >> "$GITHUB_ENV"
 
       - uses: LouisBrunner/checks-action@v2.0.0
-        if: env.run_device_test == 'true' || matrix.test.name == 'Android Render Tests'
+        if: env.run_device_test == 'true'
         id: create_check
         with:
           token: ${{ steps.generate_token.outputs.token }}
@@ -102,8 +110,8 @@ jobs:
           role-duration-seconds: 21600
           role-session-name: MySessionName
 
-      - name: Upload external data for benchmark
-        if: env.run_device_test == 'true' && matrix.test.name == 'Android Benchmark'
+      - name: Upload external data
+        if: env.run_device_test == 'true'
         run: |
           export RESULTS_API=${{ secrets.MLN_RESULTS_API }}
           export AWS_DEVICE_FARM_PROJECT_ARN=${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }}

.github/workflows/android-device-test/upload-external-data.sh

@@ -1,14 +1,16 @@
 #!/bin/bash
 
+set -e
+
 # Check if the required variables are defined
 if [ -z "$RESULTS_API" ] || [ -z "$AWS_DEVICE_FARM_PROJECT_ARN" ]; then
   echo "Error: Missing required variables."
-  usage
+  exit 1
 fi
 
 cd "$(mktemp -d)"
 
-cat << EOF > benchmark-input.json
+cat << EOF > instrumentation-test-input.json
 {
   "styleNames": ["Facebook Light", "MapTiler Basic", "Americana"],
   "styleURLs": [
@@ -16,15 +18,16 @@ cat << EOF > benchmark-input.json
     "https://api.maptiler.com/maps/basic-v2/style.json?key=get_your_own_OpIi9ZULNHzrESv6T2vL",
     "https://zelonewolf.github.io/openstreetmap-americana/style.json"
   ],
-  "resultsAPI": "$RESULTS_API"
+  "resultsAPI": "$RESULTS_API",
+  "apiKey": "get_your_own_OpIi9ZULNHzrESv6T2vL"
 }
 EOF
 
->&2 zip benchmark-input.zip benchmark-input.json
-upload_json="$(aws devicefarm create-upload --project-arn "$AWS_DEVICE_FARM_PROJECT_ARN" --type EXTERNAL_DATA --name benchmark-input.zip --output json)"
+>&2 zip instrumentation-test-input.zip instrumentation-test-input.json
+upload_json="$(aws devicefarm create-upload --project-arn "$AWS_DEVICE_FARM_PROJECT_ARN" --type EXTERNAL_DATA --name instrumentation-test-input.zip --output json)"
 upload_url="$(echo "$upload_json" | jq -r '.upload.url')"
 upload_arn="$(echo "$upload_json" | jq -r '.upload.arn')"
-curl -T benchmark-input.zip "$upload_url"
+curl -T instrumentation-test-input.zip "$upload_url"
 
 retries=5
 while true; do

platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/activity/benchmark/BenchmarkActivity.kt

@@ -146,7 +146,7 @@ class BenchmarkActivity : AppCompatActivity() {
 
     private fun getBenchmarkInputData(): BenchmarkInputData {
         // read input for benchmark from JSON file (on CI)
-        val jsonFile = File("${Environment.getExternalStorageDirectory()}/benchmark-input.json")
+        val jsonFile = File("${Environment.getExternalStorageDirectory()}/instrumentation-test-input")
         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R && Environment.isExternalStorageManager() && jsonFile.isFile) {
             val jsonFileContents = jsonFile.readText()
             val jsonElement = Json.parseToJsonElement(jsonFileContents)
@@ -192,7 +192,7 @@ class BenchmarkActivity : AppCompatActivity() {
         handler = Handler(mainLooper)
         setupToolbar()
         inputData = getBenchmarkInputData()
-        setupMapView(savedInstanceState)
+        setupMapView()
     }
 
     private fun setupToolbar() {
@@ -203,10 +203,10 @@ class BenchmarkActivity : AppCompatActivity() {
         }
     }
 
-    private fun setupMapView(savedInstanceState: Bundle?) {
+    private fun setupMapView() {
         mapView = findViewById<View>(R.id.mapView) as MapView
         if (measureFrameTime) {
-            mapView.addOnDidFinishRenderingFrameListener { fully: Boolean, frameEncodingTime: Double, frameRenderingTime: Double ->
+            mapView.addOnDidFinishRenderingFrameListener { _: Boolean, frameEncodingTime: Double, frameRenderingTime: Double ->
                 encodingTimeStore.add(frameEncodingTime * 1e3)
                 renderingTimeStore.add(frameRenderingTime * 1e3)
             }

platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/utils/ApiKeyUtils.kt

@@ -1,8 +1,25 @@
 package org.maplibre.android.testapp.utils
 
 import android.content.Context
+import android.os.Build
+import android.os.Environment
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.contentOrNull
+import kotlinx.serialization.json.jsonObject
+import kotlinx.serialization.json.jsonPrimitive
 import org.maplibre.android.MapLibre
 import java.lang.Exception
+import java.io.File
+
+fun readFromJSON(): String? {
+    val jsonFile = File("${Environment.getExternalStorageDirectory()}/instrumentation-test-input.json")
+    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R && Environment.isExternalStorageManager() && jsonFile.isFile) {
+        val jsonFileContents = jsonFile.readText()
+        val jsonElement = Json.parseToJsonElement(jsonFileContents)
+        return jsonElement.jsonObject["apiKey"]?.jsonPrimitive?.contentOrNull
+    }
+    return null
+}
 
 object ApiKeyUtils {
     /**
@@ -18,6 +35,10 @@ object ApiKeyUtils {
      * @return The api key or null if not found.
      */
     fun getApiKey(context: Context): String? {
+
+        val fromJSON = readFromJSON()
+        if (fromJSON !== null) return fromJSON
+
         return try {
             // Read out AndroidManifest
             val apiKey = MapLibre.getApiKey()