Android Benchmark on AWS Device Farm (#1925)

.github/actions/aws-device-farm-run/action.yml

@@ -19,6 +19,12 @@ inputs:
   testType:
     description: 'e.g. INSTRUMENTATION, XCTEST'
     required: true
+  testFilter:
+    description: 'Filter tests'
+    required: false
+  externalData:
+    description: 'ARN of external data package'
+    required: false
   AWS_ACCESS_KEY_ID:
     description: "AWS_ACCESS_KEY_ID"
     required: true
@@ -98,13 +104,28 @@ runs:
           done
 
       - name: Schedule test run
-        uses: realm/aws-devicefarm/test-application@master
-        with:
-          name: MapLibre Native ${{ matrix.test.name }}
-          project_arn: ${{ inputs.AWS_DEVICE_FARM_PROJECT_ARN }}
-          device_pool_arn: ${{ inputs.AWS_DEVICE_FARM_DEVICE_POOL_ARN }}
-          app_arn: ${{ env.app_arn }}
-          app_type: ${{ inputs.appType }}
-          test_type: ${{ inputs.testType }}
-          test_package_arn: ${{ env.test_package_arn }}
-          timeout: 28800
+        shell: bash
+        run: |
+          arn="$(aws devicefarm schedule-run \
+            --project-arn ${{ inputs.AWS_DEVICE_FARM_PROJECT_ARN }} \
+            --name "MapLibre Native ${{ matrix.test.name }}" \
+            --app-arn ${{ env.app_arn }} \
+            --device-pool-arn ${{ inputs.AWS_DEVICE_FARM_DEVICE_POOL_ARN }} \
+            --test type=${{ inputs.testType }},testPackageArn=${{ env.test_package_arn }}${{ inputs.testFilter && ",filter=" }}${{ inputs.testFilter }} \
+            --execution-configuration jobTimeoutMinutes=28800,videoCapture=false \
+            ${{ inputs.externalData && "--configuration extraDataPackageArn=" }}${{ inputs.externalData }} \
+            --output text --query "run.arn")"
+          
+          # wait until result is not PENDING
+          # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/devicefarm/get-run.html#output
+          while true; do
+            sleep 30
+            result="$(aws get-run --arn "$arn" --output text --query "run.result")"
+            case $result in
+              FAILED|ERRORED|STOPPED) echo "Run $result" && exit 1 ;;
+              SKIPPED|PASSED) echo "Run $result" && exit 0 ;;
+              PENDING) continue ;;
+              *) echo "Unexpected run result $result" && exit 1 ;;
+            esac
+          done
+

.github/workflows/android-device-test.yml

@@ -12,6 +12,19 @@ jobs:
       matrix:
         test: [
           {artifactName: android-render-tests, testFile: RenderTests.apk, appFile: RenderTestsApp.apk, name: "Android Render Tests"},
+          {
+            artifactName: benchmarkAPKs,
+            testFile: "androidTest/drawable/release/MapboxGLAndroidSDKTestApp-drawable-release-androidTest.apk",
+            appFile: "drawable/release/MapboxGLAndroidSDKTestApp-drawable-release.apk",
+            name: "Android Benchmark",
+            testFilter: "org.maplibre.android.benchmark.Benchmark",
+            # echo '{"styleNames": [...], "styleURLs": [...], "resultsAPI: "..." }' > benchmark-input.json 
+            # zip benchmark-input.zip benchmark-input.json 
+            # aws devicefarm create-upload --project-arn <project_arn> --type EXTERNAL_DATA --name benchmark-input.zip
+            # curl -T benchmark-input.zip <upload_url>
+            # aws devicefarm get-upload <arn>
+            externalData: "arn:aws:devicefarm:us-west-2:373521797162:upload:20687d72-0e46-403e-8f03-0941850665bc/c27174c2-63f4-4cdb-9af9-68957d75ebed"
+          }
         ]
     runs-on: ubuntu-latest
     steps:
@@ -57,6 +70,7 @@ jobs:
           AWS_ROLE_TO_ASSUME: ${{ vars.AWS_ROLE_TO_ASSUME }}
           AWS_DEVICE_FARM_PROJECT_ARN: ${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }}
           AWS_DEVICE_FARM_DEVICE_POOL_ARN: ${{ vars.AWS_DEVICE_FARM_DEVICE_POOL_ARN }}
+          externalData: ${{ matrix.test.externalData }}
 
       - uses: LouisBrunner/checks-action@v1.6.2
         if: always()

platform/android/MapboxGLAndroidSDKTestApp/src/main/java/org/maplibre/android/testapp/activity/benchmark/BenchmarkActivity.kt

@@ -1,6 +1,7 @@
 package org.maplibre.android.testapp.activity.benchmark
 
 import android.annotation.SuppressLint
+import android.app.Activity
 import android.os.Build
 import android.os.Bundle
 import android.os.Environment
@@ -295,6 +296,7 @@ class BenchmarkActivity : AppCompatActivity() {
 
     private fun benchmarkDone() {
         sendResults()
+        setResult(Activity.RESULT_OK)
         finish()
     }