Android Render Test CI (#1387)

.github/actions/aws-device-farm-run/action.yml

@@ -0,0 +1,89 @@
+name: 'aws-device-farm-run'
+description: 'Run device test on AWS Device Farm'
+inputs:
+  name:
+    description: 'Name of the test'
+    required: true
+  appType:
+    description: 'Type of upload for app'
+    required: true
+  appFile:
+    description: 'Application file to be uploaded'
+    required: false
+  testFile:
+    description: 'Application file to be uploaded'
+    required: false
+  testPackageType:
+    description: 'e.g. INSTRUMENTATION_TEST_PACKAGE, XCTEST_TEST_PACKAGE'
+    required: true
+  testType:
+    description: 'e.g. INSTRUMENTATION, XCTEST'
+    required: true
+runs:
+  using: "composite"
+  steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+          role-to-assume: ${{ vars.AWS_ROLE_TO_ASSUME }}
+          role-duration-seconds: 3600
+          role-session-name: MySessionName
+
+      - name: Create upload app
+        run: |
+          response=$(aws devicefarm create-upload --type ${{ inputs.appType }} --name ${{ inputs.appFile }} --project-arn ${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }})
+          echo "$response"
+          arn="$(jq -r '.upload.arn' <<< "$response")"
+          url="$(jq -r '.upload.url' <<< "$response")"
+          echo "app_arn=$arn" >> "$GITHUB_ENV"
+          echo "app_url=$url" >> "$GITHUB_ENV"
+
+      - name: Create upload test package
+        run: |
+          response=$(aws devicefarm create-upload --type ${{ inputs.testPackageType }} --name ${{ inputs.testFile }} --project-arn ${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }})
+          echo "$response"
+          arn="$(jq -r '.upload.arn' <<< "$response")"
+          url="$(jq -r '.upload.url' <<< "$response")"
+          echo "test_package_arn=$arn" >> "$GITHUB_ENV"
+          echo "test_package_url=$url" >> "$GITHUB_ENV"
+
+      - name: Upload ${{ inputs.appFile }}, ${{ inputs.testFile }}
+        run: |
+          curl -T ${{ inputs.appFile }} "${{ env.app_url }}"
+          curl -T ${{ inputs.testFile }} "${{ env.test_package_url }}"
+
+          max_checks=10
+          sleep_time=5
+
+          check_status() {
+            aws devicefarm get-upload --arn "$1" | jq -r '.upload.status'
+          }
+
+          while ((max_checks--)); do
+            status_app="$(check_status "${{ env.app_arn }}")"
+            status_test_package="$(check_status "${{ env.test_package_arn }}")"
+
+            echo status_app="$status_app"
+            echo status_test_package="$status_test_package"
+
+            if [[ "$status_app" == "SUCCEEDED" && "$status_test_package" == "SUCCEEDED" ]]; then
+              exit 0
+            fi
+
+            sleep $sleep_time
+          done
+
+      - name: Schedule test run
+        uses: realm/aws-devicefarm/test-application@master
+        with:
+          name: MapLibre Native ${{ matrix.test.name }}
+          project_arn: ${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }}
+          device_pool_arn: ${{ vars.AWS_DEVICE_FARM_IPHONE_DEVICE_POOL_ARN }}
+          app_arn: ${{ env.app_arn }}
+          app_type: ${{ inputs.appType }}
+          test_type: ${{ inputs.testType }}
+          test_package_arn: ${{ env.test_package_arn }}
+          timeout: 28800

.github/workflows/android-ci.yml

@@ -192,6 +192,23 @@ jobs:
           name: uploadsEnv
           path: uploads.env
 
+
+      # render test
+      - name: Build Render Test App
+        run: |
+          ./gradlew assembleDebug assembleAndroidTest
+          cp app/build/outputs/apk/debug/app-debug.apk RenderTestsApp.apk
+          cp app/build/outputs/apk/debug/app-debug.apk RenderTests.apk
+        working-directory: ./render-test/android
+
+      - name: Store Render Test .apk files
+        uses: actions/upload-artifact@v3
+        with:
+          name: android-render-tests
+          path: |
+            ./render-test/android/RenderTestsApp.apk
+            ./render-test/android/RenderTests.apk
+
       - name: Store debug artifacts
         uses: actions/upload-artifact@v3
         with:

.github/workflows/android-device-test.yml

@@ -0,0 +1,56 @@
+name: android-device-test
+
+on:
+  workflow_run:
+    workflows: [android-ci]
+    types:
+      - completed
+
+jobs:
+  create-check:
+    strategy:
+      matrix:
+        test: [
+          {artifactName: android-render-tests, testFile: RenderTests.apk, appFile: RenderTestsApp.apk, name: "Android Render Tests"},
+        ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: ./.github/actions/download-workflow-run-artifact
+        with:
+          artifact-name: ${{ matrix.test.artifactName }}
+          expect-files: "${{ matrix.test.testFile }}, ${{ matrix.test.appFile }}"
+
+      - name: Generate token
+        id: generate_token
+        uses: tibdex/github-app-token@v1
+        with:
+          app_id: ${{ secrets.MAPLIBRE_NATIVE_BOT_APP_ID }}
+          private_key: ${{ secrets.MAPLIBRE_NATIVE_BOT_PRIVATE_KEY }}
+
+      - uses: LouisBrunner/checks-action@v1.6.2
+        id: create_check
+        with:
+          token: ${{ steps.generate_token.outputs.token }}
+          details_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          status: queued
+          name: ${{ matrix.test.name }}
+          sha: ${{ github.event.workflow_run.head_sha }}
+
+      - uses: ./.github/actions/aws-device-farm-run
+        with:
+          name: ${{ matrix.test.name }}
+          appType: ANDROID_APP
+          appFile: ${{ matrix.test.appFile }}
+          testFile: ${{ matrix.test.appFile }}
+          testPackageType: INSTRUMENTATION_TEST_PACKAGE
+          testType: INSTRUMENTATION
+
+      - uses: LouisBrunner/checks-action@v1.6.2
+        if: always()
+        with:
+          token: ${{ steps.generate_token.outputs.token }}
+          check_id: ${{ steps.create_check.outputs.check_id }}
+          conclusion: ${{ job.status }}
+          sha: ${{ github.event.workflow_run.sha }}

metrics/android-render-test-runner-metrics.json

@@ -15,6 +15,5 @@
         "probeGFX",
         "probeMemory",
         "probeNetwork"
-    ],
-    "result_path": "/sdcard"
+    ]
 }

metrics/android-render-test-runner-style.json

@@ -14,6 +14,5 @@
     "probes": [
         "probeGFX",
         "probeNetwork"
-    ],
-    "result_path": "/sdcard"
+    ]
 }

metrics/ignores/platform-android.json

@@ -10,5 +10,10 @@
     "render-tests/regressions/mapbox-gl-native#9976": "Needs to be investigated and fixed.",
     "render-tests/runtime-styling/image-add-pattern": "Needs to be investigated and fixed.",
     "render-tests/runtime-styling/image-update-pattern": "Needs to be investigated and fixed.",
-    "render-tests/text-variable-anchor/left-top-right-bottom-offset-tile-map-mode": "https://github.com/mapbox/api-gl/issues/1265"
+    "render-tests/text-variable-anchor/left-top-right-bottom-offset-tile-map-mode": "Needs to be investigated and fixed.",
+    "render-tests/regressions/mapbox-gl-js#2305": "Needs to be investigated and fixed.",
+    "render-tests/regressions/mapbox-gl-js#5978": "Needs to be investigated and fixed.",
+    "render-tests/sprites/1x-screen-2x-pattern": "Needs to be investigated and fixed.",
+    "render-tests/text-field/formatted-images": "Needs to be investigated and fixed.",
+    "render-tests/symbol-visibility/visible": "Needs to be investigated and fixed."
 }

platform/android/src/test/render_test_runner.cpp

@@ -18,8 +18,8 @@ void android_main(struct android_app* app) {
     int outFd, outEvents;
     struct android_poll_source* source = nullptr;
     if (!copyFile(env, app->activity->assetManager, zipFile, storagePath, "data.zip")) {
-        mbgl::Log::Error(
-            mbgl::Event::General, "Failed to copy zip File '%s' to external storage for upzipping", zipFile.c_str());
+        mbgl::Log::Error(mbgl::Event::General,
+                         "Failed to copy zip File '" + zipFile + "' to external storage for upzipping");
     } else {
         unZipFile(env, zipFile, storagePath);
 
@@ -41,12 +41,12 @@ void android_main(struct android_app* app) {
                     source->process(app, source);
                 }
 
-                mbgl::Log::Info(mbgl::Event::General, "Current finished tests number is '%d' ", ++finishedTestCount);
+                mbgl::Log::Info(mbgl::Event::General,
+                                "Current finished tests number is '" + std::to_string(++finishedTestCount) + "'");
             };
-            mbgl::Log::Info(
-                mbgl::Event::General, "Start running RenderTestRunner with manifest: '%s'", manifest.c_str());
+            mbgl::Log::Info(mbgl::Event::General, "Start running RenderTestRunner with manifest: '" + manifest + "'");
             bool result = mbgl::runRenderTests(argv.size() - 1, argv.data(), testStatus) == 0;
-            mbgl::Log::Info(mbgl::Event::General, "End running RenderTestRunner with manifest: '%s'", manifest.c_str());
+            mbgl::Log::Info(mbgl::Event::General, "End running RenderTestRunner with manifest: '" + manifest + "'");
             return result;
         };
 

render-test/android/.gitignore

@@ -0,0 +1,4 @@
+.cxx
+.gradle
+app/build
+app/src/main/assets/data.zip

render-test/android/app/build.gradle

@@ -43,6 +43,8 @@ android {
             signingConfig signingConfigs.debug
         }
     }
+
+    namespace 'android.app'
 }
 
 dependencies {

render-test/android/app/src/main/AndroidManifest.xml

@@ -1,6 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
-          package="com.example.native_activity"
           android:versionCode="1"
           android:versionName="1.0">
   <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
@@ -13,7 +12,8 @@
 
     <activity android:name="android.app.NativeActivity"
               android:label="@string/app_name"
-              android:screenOrientation="portrait">
+              android:screenOrientation="portrait"
+              android:exported="true">
       <meta-data android:name="android.app.lib_name"
                  android:value="mbgl-render-test-runner" />
       <intent-filter>

render-test/android/build.gradle

@@ -4,7 +4,7 @@ buildscript {
        jcenter()
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:3.5.1'
+        classpath 'com.android.tools.build:gradle:8.0.2'
     }
 }
 

render-test/android/gradle.properties

@@ -1 +1,2 @@
-org.gradle.jvmargs=-Xmx1536m
+org.gradle.jvmargs=-Xmx1536m
+android.useAndroidX=true