From 8ad141e213e739fe38af54292279cd69ac8610f2 Mon Sep 17 00:00:00 2001 From: Bart Louwers Date: Mon, 22 Apr 2024 22:47:22 +0200 Subject: [PATCH] Run Android Instrumentation Tests as part of PR workflow (#2309) --- .../actions/aws-device-farm-run/action.yml | 1 + .github/workflows/android-ci.yml | 104 ++---------------- .github/workflows/android-device-test.yml | 16 ++- .../upload-external-data.sh | 15 ++- .../activity/benchmark/BenchmarkActivity.kt | 8 +- .../android/testapp/utils/ApiKeyUtils.kt | 21 ++++ 6 files changed, 55 insertions(+), 110 deletions(-) diff --git a/.github/actions/aws-device-farm-run/action.yml b/.github/actions/aws-device-farm-run/action.yml index bd62cf98c..da140eff7 100644 --- a/.github/actions/aws-device-farm-run/action.yml +++ b/.github/actions/aws-device-farm-run/action.yml @@ -121,6 +121,7 @@ runs: --device-pool-arn ${{ inputs.AWS_DEVICE_FARM_DEVICE_POOL_ARN }} \ --test type=${{ inputs.testType }},testPackageArn=${{ env.test_package_arn }}${{ inputs.testFilter && ',filter=' }}${{ inputs.testFilter }}${{ inputs.testSpecArn && ',testSpecArn=' }}${{ inputs.testSpecArn }} \ ${{ inputs.externalData && '--configuration extraDataPackageArn=' }}${{ inputs.externalData }} \ + --execution-configuration videoCapture=false \ --output text --query "run.arn")" echo "runArn=$arn" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/android-ci.yml b/.github/workflows/android-ci.yml index 8d1482288..fc6a832a2 100644 --- a/.github/workflows/android-ci.yml +++ b/.github/workflows/android-ci.yml @@ -137,72 +137,20 @@ jobs: - if: github.event_name == 'pull_request' uses: ./.github/actions/save-pr-number - - name: Build UI tests - if: github.ref == 'refs/heads/main' + - name: Build Instrumentation Tests, copy to platform/android run: | ./gradlew assembleLegacyDebug assembleLegacyDebugAndroidTest -PtestBuildType=debug + cp MapLibreAndroidTestApp/build/outputs/apk/legacy/debug/MapLibreAndroidTestApp-legacy-debug.apk InstrumentationTestApp.apk + cp MapLibreAndroidTestApp/build/outputs/apk/androidTest/legacy/debug/MapLibreAndroidTestApp-legacy-debug-androidTest.apk InstrumentationTests.apk - - name: Configure AWS Credentials - if: github.ref == 'refs/heads/main' - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - role-to-assume: ${{ vars.AWS_ROLE_TO_ASSUME }} - role-duration-seconds: 21600 - role-session-name: MySessionName - - - name: Create upload - if: github.ref == 'refs/heads/main' - uses: realm/aws-devicefarm/create-upload@master - id: upload-android-app - with: - project_arn: ${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }} - name: MapLibreAndroidTestApp-debug.apk - type: ANDROID_APP - - - name: Create upload - if: github.ref == 'refs/heads/main' - uses: realm/aws-devicefarm/create-upload@master - id: upload-android-test - with: - project_arn: ${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }} - name: MapLibreAndroidTestApp-debug-androidTest.apk - type: INSTRUMENTATION_TEST_PACKAGE - - - name: Upload Android UI test - if: github.ref == 'refs/heads/main' - run: | - curl -T MapLibreAndroidTestApp/build/outputs/apk/legacy/debug/MapLibreAndroidTestApp-legacy-debug.apk '${{ steps.upload-android-app.outputs.url }}' - curl -T MapLibreAndroidTestApp/build/outputs/apk/androidTest/legacy/debug/MapLibreAndroidTestApp-legacy-debug-androidTest.apk '${{ steps.upload-android-test.outputs.url }}' - - - name: Write uploads.env - if: github.ref == 'refs/heads/main' - working-directory: . - run: | - echo "ANDROID_APP_ARN=${{ steps.upload-android-app.outputs.arn }}" >> uploads.env - echo "ANDROID_TEST_ARN=${{ steps.upload-android-test.outputs.arn }}" >> uploads.env - - - uses: actions/upload-artifact@v4 - if: github.ref == 'refs/heads/main' - with: - if-no-files-found: error - name: uploadsEnv - path: uploads.env - - - name: Store debug artifacts + - name: Upload android-ui-test uses: actions/upload-artifact@v4 with: - name: debug-artifacts + if-no-files-found: error + name: android-ui-test path: | - MapLibreAndroidTestApp/build/outputs/apk/debug - MapLibreAndroid/build/reports/lint-results.html - MapLibreAndroid/lint-baseline.xml - MapLibreAndroidTestApp/build/reports/lint-results.html - MapLibreAndroidTestApp/build/reports/lint-results.xml - MapLibreAndroidTestApp/lint-baseline.xml - MapLibreAndroid/build/intermediates/cmake/debug/obj + platform/android/InstrumentationTestApp.apk + platform/android/InstrumentationTests.apk android-build-render-test: runs-on: ubuntu-latest @@ -240,42 +188,6 @@ jobs: ./render-test/android/RenderTestsApp.apk ./render-test/android/RenderTests.apk - android-instrumentation-test: - needs: android-build - - runs-on: ubuntu-latest - if: github.repository_owner == 'maplibre' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/android-annotations') - steps: - - uses: actions/download-artifact@v4 - with: - name: uploadsEnv - - - name: Read uploads.env - run: cat uploads.env >> "$GITHUB_ENV" - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - role-to-assume: ${{ vars.AWS_ROLE_TO_ASSUME }} - role-duration-seconds: 21600 - role-session-name: MySessionName - - - name: Schedule test run - uses: realm/aws-devicefarm/test-application@master - with: - name: MapLibre Native Android Instrumentation Test - project_arn: ${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }} - device_pool_arn: ${{ vars.AWS_DEVICE_FARM_DEVICE_POOL_ARN }} - app_arn: ${{ env.ANDROID_APP_ARN }} - test_spec_arn: ${{ vars.AWS_DEVICE_FARM_TEST_SPEC_ARN_ANDROID_UI_TESTS }} - app_type: ANDROID_APP - test_type: INSTRUMENTATION - test_package_arn: ${{ env.ANDROID_TEST_ARN }} - timeout: 28800 - android-ci-result: runs-on: ubuntu-latest if: needs.pre_job.outputs.should_skip != 'true' && always() diff --git a/.github/workflows/android-device-test.yml b/.github/workflows/android-device-test.yml index f8da7bab2..498fd52ce 100644 --- a/.github/workflows/android-device-test.yml +++ b/.github/workflows/android-device-test.yml @@ -10,7 +10,6 @@ jobs: android-device-test: strategy: fail-fast: false - max-parallel: 2 matrix: test: [ { @@ -32,6 +31,15 @@ jobs: # benchmark-android.yaml # see https://github.com/maplibre/ci-runners/tree/main/aws-device-farm/custom-test-envs testSpecArn: "arn:aws:devicefarm:us-west-2:373521797162:upload:20687d72-0e46-403e-8f03-0941850665bc/14862afb-cf88-44aa-9f1e-5131cbb22f01" + }, + { + artifactName: android-ui-test, + testFile: InstrumentationTests.apk, + appFile: InstrumentationTestApp.apk, + name: "Android Instrumentation Tests", + # Google Pixel 7 Pro + devicePool: "arn:aws:devicefarm:us-west-2:373521797162:devicepool:20687d72-0e46-403e-8f03-0941850665bc/9692fe7f-86a9-4ecc-908f-175600968564", + testSpecArn: "arn:aws:devicefarm:us-west-2:373521797162:upload:20687d72-0e46-403e-8f03-0941850665bc/09e0738e-c91e-4c5f-81e6-06a06cc340d8" } ] runs-on: ubuntu-latest @@ -76,7 +84,7 @@ jobs: echo "run_device_test=true" >> "$GITHUB_ENV" - uses: LouisBrunner/checks-action@v2.0.0 - if: env.run_device_test == 'true' || matrix.test.name == 'Android Render Tests' + if: env.run_device_test == 'true' id: create_check with: token: ${{ steps.generate_token.outputs.token }} @@ -102,8 +110,8 @@ jobs: role-duration-seconds: 21600 role-session-name: MySessionName - - name: Upload external data for benchmark - if: env.run_device_test == 'true' && matrix.test.name == 'Android Benchmark' + - name: Upload external data + if: env.run_device_test == 'true' run: | export RESULTS_API=${{ secrets.MLN_RESULTS_API }} export AWS_DEVICE_FARM_PROJECT_ARN=${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }} diff --git a/.github/workflows/android-device-test/upload-external-data.sh b/.github/workflows/android-device-test/upload-external-data.sh index 5610e623c..a3efb84d2 100755 --- a/.github/workflows/android-device-test/upload-external-data.sh +++ b/.github/workflows/android-device-test/upload-external-data.sh @@ -1,14 +1,16 @@ #!/bin/bash +set -e + # Check if the required variables are defined if [ -z "$RESULTS_API" ] || [ -z "$AWS_DEVICE_FARM_PROJECT_ARN" ]; then echo "Error: Missing required variables." - usage + exit 1 fi cd "$(mktemp -d)" -cat << EOF > benchmark-input.json +cat << EOF > instrumentation-test-input.json { "styleNames": ["Facebook Light", "MapTiler Basic", "Americana"], "styleURLs": [ @@ -16,15 +18,16 @@ cat << EOF > benchmark-input.json "https://api.maptiler.com/maps/basic-v2/style.json?key=get_your_own_OpIi9ZULNHzrESv6T2vL", "https://zelonewolf.github.io/openstreetmap-americana/style.json" ], - "resultsAPI": "$RESULTS_API" + "resultsAPI": "$RESULTS_API", + "apiKey": "get_your_own_OpIi9ZULNHzrESv6T2vL" } EOF ->&2 zip benchmark-input.zip benchmark-input.json -upload_json="$(aws devicefarm create-upload --project-arn "$AWS_DEVICE_FARM_PROJECT_ARN" --type EXTERNAL_DATA --name benchmark-input.zip --output json)" +>&2 zip instrumentation-test-input.zip instrumentation-test-input.json +upload_json="$(aws devicefarm create-upload --project-arn "$AWS_DEVICE_FARM_PROJECT_ARN" --type EXTERNAL_DATA --name instrumentation-test-input.zip --output json)" upload_url="$(echo "$upload_json" | jq -r '.upload.url')" upload_arn="$(echo "$upload_json" | jq -r '.upload.arn')" -curl -T benchmark-input.zip "$upload_url" +curl -T instrumentation-test-input.zip "$upload_url" retries=5 while true; do diff --git a/platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/activity/benchmark/BenchmarkActivity.kt b/platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/activity/benchmark/BenchmarkActivity.kt index b67532a10..9559a4982 100644 --- a/platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/activity/benchmark/BenchmarkActivity.kt +++ b/platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/activity/benchmark/BenchmarkActivity.kt @@ -146,7 +146,7 @@ class BenchmarkActivity : AppCompatActivity() { private fun getBenchmarkInputData(): BenchmarkInputData { // read input for benchmark from JSON file (on CI) - val jsonFile = File("${Environment.getExternalStorageDirectory()}/benchmark-input.json") + val jsonFile = File("${Environment.getExternalStorageDirectory()}/instrumentation-test-input") if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R && Environment.isExternalStorageManager() && jsonFile.isFile) { val jsonFileContents = jsonFile.readText() val jsonElement = Json.parseToJsonElement(jsonFileContents) @@ -192,7 +192,7 @@ class BenchmarkActivity : AppCompatActivity() { handler = Handler(mainLooper) setupToolbar() inputData = getBenchmarkInputData() - setupMapView(savedInstanceState) + setupMapView() } private fun setupToolbar() { @@ -203,10 +203,10 @@ class BenchmarkActivity : AppCompatActivity() { } } - private fun setupMapView(savedInstanceState: Bundle?) { + private fun setupMapView() { mapView = findViewById(R.id.mapView) as MapView if (measureFrameTime) { - mapView.addOnDidFinishRenderingFrameListener { fully: Boolean, frameEncodingTime: Double, frameRenderingTime: Double -> + mapView.addOnDidFinishRenderingFrameListener { _: Boolean, frameEncodingTime: Double, frameRenderingTime: Double -> encodingTimeStore.add(frameEncodingTime * 1e3) renderingTimeStore.add(frameRenderingTime * 1e3) } diff --git a/platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/utils/ApiKeyUtils.kt b/platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/utils/ApiKeyUtils.kt index 7e85c8ea6..43e2b7508 100644 --- a/platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/utils/ApiKeyUtils.kt +++ b/platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/utils/ApiKeyUtils.kt @@ -1,8 +1,25 @@ package org.maplibre.android.testapp.utils import android.content.Context +import android.os.Build +import android.os.Environment +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive import org.maplibre.android.MapLibre import java.lang.Exception +import java.io.File + +fun readFromJSON(): String? { + val jsonFile = File("${Environment.getExternalStorageDirectory()}/instrumentation-test-input.json") + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R && Environment.isExternalStorageManager() && jsonFile.isFile) { + val jsonFileContents = jsonFile.readText() + val jsonElement = Json.parseToJsonElement(jsonFileContents) + return jsonElement.jsonObject["apiKey"]?.jsonPrimitive?.contentOrNull + } + return null +} object ApiKeyUtils { /** @@ -18,6 +35,10 @@ object ApiKeyUtils { * @return The api key or null if not found. */ fun getApiKey(context: Context): String? { + + val fromJSON = readFromJSON() + if (fromJSON !== null) return fromJSON + return try { // Read out AndroidManifest val apiKey = MapLibre.getApiKey()