Android Instrumentation Tests Improvements (#2321)

.github/workflows/android-device-test.yml

@@ -61,6 +61,7 @@ jobs:
         id: generate_token
         uses: tibdex/github-app-token@v2
         with:
+          revoke: false  # revoking will fail for long running workflows, because the token will already have expired
           app_id: ${{ secrets.MAPLIBRE_NATIVE_BOT_APP_ID }}
           private_key: ${{ secrets.MAPLIBRE_NATIVE_BOT_PRIVATE_KEY }}
 
@@ -103,12 +104,9 @@ jobs:
         if: env.run_device_test == 'true'
         uses: aws-actions/configure-aws-credentials@v4
         with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: us-west-2
-          role-to-assume: ${{ vars.AWS_ROLE_TO_ASSUME }}
+          role-to-assume: ${{ vars.OIDC_AWS_ROLE_TO_ASSUME }}
-          role-duration-seconds: 21600
+          role-session-name: ${{ github.run_id }}
-          role-session-name: MySessionName
 
       - name: Upload external data
         if: env.run_device_test == 'true'
@@ -143,10 +141,24 @@ jobs:
           npm install --ignore-scripts
           node scripts/aws-device-farm/download-benchmark-results.mjs ${{ steps.aws_device_farm_run.outputs.runArn }}
 
-      - uses: LouisBrunner/checks-action@v2.0.0
+      - name: Log Test Spec Output
-        if: always() && (env.run_device_test == 'true' || matrix.test.name == 'Android Render Tests')
+        if: failure() && env.run_device_test == 'true'
+        run: |
+          npm install --ignore-scripts
+          node scripts/aws-device-farm/log-test-spec-output.mjs ${{ steps.aws_device_farm_run.outputs.runArn }}
+
+      - name: Generate another token (previous one could have expired)
+        if: always() && env.run_device_test == 'true'
+        id: generate_token_2
+        uses: tibdex/github-app-token@v2
         with:
-          token: ${{ steps.generate_token.outputs.token }}
+          app_id: ${{ secrets.MAPLIBRE_NATIVE_BOT_APP_ID }}
+          private_key: ${{ secrets.MAPLIBRE_NATIVE_BOT_PRIVATE_KEY }}
+
+      - uses: LouisBrunner/checks-action@v2.0.0
+        if: always() && env.run_device_test == 'true'
+        with:
+          token: ${{ steps.generate_token_2.outputs.token }}
           check_id: ${{ steps.create_check.outputs.check_id }}
           conclusion: ${{ job.status }}
           sha: ${{ github.event.workflow_run.sha }}

platform/android/MapLibreAndroid/src/main/java/org/maplibre/android/MapLibre.java

@@ -128,7 +128,7 @@ public final class MapLibre {
    */
   @Nullable
   public static String getApiKey() {
-    validateMapbox();
+    validateMapLibre();
     return INSTANCE.apiKey;
   }
 
@@ -136,7 +136,7 @@ public final class MapLibre {
    * Set the current active apiKey.
    */
   public static void setApiKey(String apiKey) {
-    validateMapbox();
+    validateMapLibre();
     throwIfApiKeyInvalid(apiKey);
     INSTANCE.apiKey = apiKey;
     FileSource.getInstance(getApplicationContext()).setApiKey(apiKey);
@@ -147,18 +147,18 @@ public final class MapLibre {
    */
   @Nullable
   public static TileServerOptions getTileServerOptions() {
-    validateMapbox();
+    validateMapLibre();
     return INSTANCE.tileServerOptions;
   }
 
   /**
    * Get all pre-defined styles
    *
-   * @return List of predefined styles
+   * @return Array of predefined styles
    */
   @Nullable
   public static DefaultStyle[] getPredefinedStyles() {
-    validateMapbox();
+    validateMapLibre();
     if (INSTANCE.tileServerOptions != null) {
       return INSTANCE.tileServerOptions.getDefaultStyles();
     }
@@ -172,7 +172,7 @@ public final class MapLibre {
    */
   @Nullable
   public static DefaultStyle getPredefinedStyle(String name) {
-    validateMapbox();
+    validateMapLibre();
     if (INSTANCE.tileServerOptions != null) {
       DefaultStyle[] styles = INSTANCE.tileServerOptions.getDefaultStyles();
       for (DefaultStyle style : styles) {
@@ -191,7 +191,7 @@ public final class MapLibre {
    */
   @NonNull
   public static Context getApplicationContext() {
-    validateMapbox();
+    validateMapLibre();
     return INSTANCE.context;
   }
 
@@ -203,7 +203,7 @@ public final class MapLibre {
    *                  disconnected, and null for ConnectivityManager to determine.
    */
   public static synchronized void setConnected(Boolean connected) {
-    validateMapbox();
+    validateMapLibre();
     ConnectivityReceiver.instance(INSTANCE.context).setConnected(connected);
   }
 
@@ -214,7 +214,7 @@ public final class MapLibre {
    * @return true if there is an internet connection, false otherwise
    */
   public static synchronized Boolean isConnected() {
-    validateMapbox();
+    validateMapLibre();
     return ConnectivityReceiver.instance(INSTANCE.context).isConnected();
   }
 
@@ -242,7 +242,7 @@ public final class MapLibre {
   /**
    * Runtime validation of MapLibre creation.
    */
-  private static void validateMapbox() {
+  private static void validateMapLibre() {
     if (INSTANCE == null) {
       throw new MapLibreConfigurationException();
     }
@@ -260,7 +260,7 @@ public final class MapLibre {
     }
 
     apiKey = apiKey.trim().toLowerCase(MapLibreConstants.MAPLIBRE_LOCALE);
-    return apiKey.length() != 0;
+    return !apiKey.isEmpty();
   }
 
   /**

platform/android/MapLibreAndroidTestApp/src/androidTest/java/org/maplibre/android/integration/ViewPagerScrollTest.kt

@@ -4,6 +4,7 @@ import androidx.test.filters.LargeTest
 import androidx.test.internal.runner.junit4.AndroidJUnit4ClassRunner
 import androidx.test.rule.ActivityTestRule
 import androidx.test.uiautomator.UiSelector
+import org.junit.Ignore
 import org.maplibre.android.testapp.activity.fragment.ViewPagerActivity
 import org.junit.Rule
 import org.junit.Test
@@ -12,6 +13,7 @@ import org.junit.runner.RunWith
 /**
  * Regression test that validates MapFragment integration with a ViewPager
  */
+@Ignore("https://github.com/maplibre/maplibre-native/issues/2316")
 @RunWith(AndroidJUnit4ClassRunner::class)
 class ViewPagerScrollTest : BaseIntegrationTest() {
 

platform/android/MapLibreAndroidTestApp/src/androidTest/java/org/maplibre/android/maps/MapGestureDetectorTest.kt

@@ -187,7 +187,7 @@ class MapGestureDetectorTest : BaseTest() {
         onView(withId(R.id.mapView)).perform(quickScale(-300f, withVelocity = false, duration = 750L))
 
         rule.runOnUiThread {
-            Assert.assertEquals(3.0, maplibreMap.cameraPosition.zoom, 0.001)
+            Assert.assertEquals(3.0, maplibreMap.cameraPosition.zoom, 0.01)
         }
     }
 }

platform/android/MapLibreAndroidTestApp/src/androidTest/java/org/maplibre/android/maps/MapLibreTest.kt

@@ -3,12 +3,12 @@ package org.maplibre.android.maps
 import androidx.test.annotation.UiThreadTest
 import androidx.test.internal.runner.junit4.AndroidJUnit4ClassRunner
 import org.junit.After
-import org.junit.Assert.*
+import org.junit.Assert
 import org.junit.Before
 import org.junit.Test
 import org.junit.runner.RunWith
 import org.maplibre.android.AppCenter
-import org.maplibre.android.MapLibre.*
+import org.maplibre.android.MapLibre
 import org.maplibre.android.exceptions.MapLibreConfigurationException
 
 @RunWith(AndroidJUnit4ClassRunner::class)
@@ -16,43 +16,48 @@ class MapLibreTest : AppCenter() {
     private var realToken: String? = null
     @Before
     fun setup() {
-        realToken = getApiKey()
+        realToken = MapLibre.getApiKey()
     }
 
     @Test
     @UiThreadTest
     fun testConnected() {
-        assertTrue(isConnected())
+        Assert.assertTrue(MapLibre.isConnected())
 
         // test manual connectivity
-        setConnected(true)
+        MapLibre.setConnected(true)
-        assertTrue(isConnected())
+        Assert.assertTrue(MapLibre.isConnected())
-        setConnected(false)
+        MapLibre.setConnected(false)
-        assertFalse(isConnected())
+        Assert.assertFalse(MapLibre.isConnected())
 
         // reset to Android connectivity
-        setConnected(null)
+        MapLibre.setConnected(null)
-        assertTrue(isConnected())
+        Assert.assertTrue(MapLibre.isConnected())
     }
 
     @Test
     @UiThreadTest
     fun setApiKey() {
-        setApiKey(API_KEY)
+        MapLibre.setApiKey(API_KEY)
-        assertSame(API_KEY, getApiKey())
+        Assert.assertSame(API_KEY, MapLibre.getApiKey())
-        setApiKey(API_KEY_2)
+        MapLibre.setApiKey(API_KEY_2)
-        assertSame(API_KEY_2, getApiKey())
+        Assert.assertSame(API_KEY_2, MapLibre.getApiKey())
     }
 
     @Test
     @UiThreadTest
     fun setNullApiKey() {
-        assertThrows(MapLibreConfigurationException::class.java) { setApiKey(null) }
+        Assert.assertThrows(
+            MapLibreConfigurationException::class.java
+        ) { MapLibre.setApiKey(null) }
     }
 
     @After
     fun tearDown() {
-        setApiKey(realToken)
+        if (realToken?.isNotEmpty() == true) {
+            MapLibre.setApiKey(realToken)
+        }
+
     }
 
     companion object {

platform/android/MapLibreAndroidTestApp/src/androidTest/java/org/maplibre/android/offline/OfflineDownloadTest.kt

@@ -2,6 +2,7 @@ package org.maplibre.android.offline
 
 import androidx.test.internal.runner.junit4.AndroidJUnit4ClassRunner
 import androidx.test.rule.ActivityTestRule
+import org.junit.Ignore
 import org.maplibre.geojson.Point
 import org.maplibre.android.log.Logger
 import org.maplibre.android.maps.Style
@@ -16,6 +17,7 @@ import java.util.concurrent.TimeoutException
 /**
  * Integration test that validates downloading an offline region from a point geometry at zoomlevel 17
  */
+@Ignore("https://github.com/maplibre/maplibre-native/issues/2318")
 @RunWith(AndroidJUnit4ClassRunner::class)
 class OfflineDownloadTest : OfflineRegion.OfflineRegionObserver {
 

platform/android/MapLibreAndroidTestApp/src/androidTest/java/org/maplibre/android/snapshotter/MapSnapshotterTest.kt

@@ -10,6 +10,7 @@ import org.maplibre.android.style.layers.PropertyFactory
 import org.maplibre.android.testapp.activity.FeatureOverviewActivity
 import org.junit.Assert
 import org.junit.Assert.assertNotNull
+import org.junit.Ignore
 import org.junit.Rule
 import org.junit.Test
 import org.junit.runner.RunWith
@@ -20,6 +21,7 @@ import java.util.concurrent.TimeoutException
 /**
  * Integration test that validates if a snapshotter creation
  */
+@Ignore("https://github.com/maplibre/maplibre-native/issues/2317")
 @RunWith(AndroidJUnit4ClassRunner::class)
 class MapSnapshotterTest {
 

platform/android/MapLibreAndroidTestApp/src/androidTest/java/org/maplibre/android/testapp/geometry/LatLngBoundsTest.java

@@ -2,6 +2,7 @@ package org.maplibre.android.testapp.geometry;
 
 import static org.junit.Assert.assertEquals;
 
+import org.junit.Ignore;
 import org.maplibre.android.camera.CameraUpdateFactory;
 import org.maplibre.android.geometry.LatLng;
 import org.maplibre.android.geometry.LatLngBounds;
@@ -15,6 +16,7 @@ import org.junit.Test;
 /**
  * Instrumentation test to validate integration of LatLngBounds
  */
+@Ignore("https://github.com/maplibre/maplibre-native/issues/2319")
 public class LatLngBoundsTest extends BaseTest {
 
   private static final double MAP_BEARING = 50;

platform/android/MapLibreAndroidTestApp/src/androidTest/java/org/maplibre/android/testapp/maps/StyleLoadTest.kt

@@ -23,7 +23,7 @@ class StyleLoadTest : EspressoTest() {
             val layer = SymbolLayer("id", "id")
             maplibreMap.setStyle(Style.Builder().withSource(source).withLayer(layer))
             TestingAsyncUtils.waitForLayer(uiController, mapView)
-            maplibreMap.setStyle(Style.Builder().fromUrl(Style.getPredefinedStyle("Streets")))
+            maplibreMap.setStyle(Style.getPredefinedStyles()[0].url)
             TestingAsyncUtils.waitForLayer(uiController, mapView)
             source.setGeoJson("{}")
         }

platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/activity/espresso/DeviceIndependentTestActivity.kt

@@ -3,15 +3,14 @@ package org.maplibre.android.testapp.activity.espresso
 import android.os.Bundle
 import androidx.appcompat.app.AppCompatActivity
 import org.maplibre.android.maps.MapView
-import org.maplibre.android.maps.MapLibreMap
-import org.maplibre.android.maps.OnMapReadyCallback
 import org.maplibre.android.maps.Style
 import org.maplibre.android.testapp.R
+import timber.log.Timber
+
+class DeviceIndependentTestActivity : AppCompatActivity() {
+    private val TAG = "DeviceIndependentTestActivity"
 
-class DeviceIndependentTestActivity : AppCompatActivity(), OnMapReadyCallback {
     lateinit var mapView: MapView
-    lateinit var maplibreMap: MapLibreMap
-        protected set
 
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
@@ -20,12 +19,10 @@ class DeviceIndependentTestActivity : AppCompatActivity(), OnMapReadyCallback {
         // Initialize map as normal
         mapView = findViewById(R.id.mapView)
         mapView.onCreate(savedInstanceState)
-        mapView.getMapAsync(this)
+        mapView.getMapAsync {
-    }
+            Timber.i(TAG, "Setting style")
-
+            it.setStyle(Style.getPredefinedStyles()[0].url)
-    override fun onMapReady(map: MapLibreMap) {
+        }
-        maplibreMap = map
-        maplibreMap.setStyle(Style.getPredefinedStyle("Streets"))
     }
 
     public override fun onResume() {

platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/activity/espresso/PixelTestActivity.kt

@@ -26,7 +26,8 @@ class PixelTestActivity : AppCompatActivity(), OnMapReadyCallback {
 
     override fun onMapReady(map: MapLibreMap) {
         maplibreMap = map
-        maplibreMap.setStyle(Style.getPredefinedStyle("Streets"))
+        val styleURL = Style.getPredefinedStyles()[0].url
+        maplibreMap.setStyle(styleURL)
     }
 
     public override fun onResume() {

platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/activity/offline/OfflineActivity.kt

@@ -45,6 +45,8 @@ class OfflineActivity : AppCompatActivity(), DownloadRegionDialogListener {
     private var downloadRegion: Button? = null
     private var listRegions: Button? = null
     private var isEndNotified = false
+    val STYLE_URL: String
+        get() = Style.getPredefinedStyle("Streets")
 
     /*
    * Offline objects
@@ -312,8 +314,5 @@ class OfflineActivity : AppCompatActivity(), DownloadRegionDialogListener {
         // JSON encoding/decoding
         val JSON_CHARSET = Charsets.UTF_8
         const val JSON_FIELD_REGION_NAME = "FIELD_REGION_NAME"
-
-        // Style URL
-        val STYLE_URL = Style.getPredefinedStyle("Streets")
     }
 }

platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/activity/style/GridSourceActivity.kt

@@ -100,7 +100,7 @@ class GridSourceActivity : AppCompatActivity(), OnMapReadyCallback {
         )
         map.setStyle(
             Style.Builder()
-                .fromUri(Style.getPredefinedStyle("Streets"))
+                .fromUri(Style.getPredefinedStyles()[0].url)
                 .withLayer(layer!!)
                 .withSource(source!!)
         )

scripts/aws-device-farm/device-farm-client.mjs

@@ -0,0 +1,7 @@
+import {
+  DeviceFarmClient,
+} from "@aws-sdk/client-device-farm";
+
+export function getDeviceFarmClient() {
+  return new DeviceFarmClient({ region: "us-west-2" });
+}

scripts/aws-device-farm/download-benchmark-results.mjs

@@ -6,11 +6,12 @@ import https from "node:https";
 import readline from "node:readline";
 
 import {
-  DeviceFarmClient,
   ListJobsCommand,
   ListArtifactsCommand,
 } from "@aws-sdk/client-device-farm";
-import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3";
+import { PutObjectCommand } from "@aws-sdk/client-s3";
+import { getDeviceFarmClient } from "./device-farm-client.mjs";
+import { getS3Client } from "./s3-client.mjs";
 
 /**
  * Retrieves a log file and looks for the line that includes the benchmark results.
@@ -46,7 +47,7 @@ function usage() {
   process.exit(1);
 }
 
-const deviceFarmClient = new DeviceFarmClient({ region: "us-west-2" });
+const deviceFarmClient = getDeviceFarmClient();
 
 if (process.argv.length !== 3) usage();
 
@@ -103,9 +104,7 @@ const benchmarkResults = benchmarkResultLogLines
   .filter((line) => !!line)
   .map((jsonVal) => JSON.parse(jsonVal));
 
-const s3Client = new S3Client({
+const s3Client = getS3Client();
-  region: "eu-central-1",
-});
 
 // store benchmark results as JSON files on S3
 await Promise.all(

scripts/aws-device-farm/log-test-spec-output.mjs

@@ -0,0 +1,47 @@
+import { ListArtifactsCommand } from "@aws-sdk/client-device-farm";
+import { getDeviceFarmClient } from "./device-farm-client.mjs";
+
+/**
+ * @returns {never}
+ */
+function usage() {
+  console.error("Logs test spec output from AWS Device Farm");
+  console.error(`Usage: node ${process.argv.at(1)} RUN_ARN`);
+  process.exit(1);
+}
+
+if (process.argv.length !== 3) usage();
+
+const arn = process.argv.at(2);
+
+if (typeof arn !== 'string') usage();
+
+const deviceFarmClient = getDeviceFarmClient();
+
+/**
+ * Looks for the run with the provided ARN and returns the test spec output.
+ * 
+ * @param {string} arn 
+ * @returns string
+ */
+async function getTestSpecOutput(arn) {
+  const logs = await deviceFarmClient.send(new ListArtifactsCommand({
+    arn,
+    type: 'FILE'
+  }));
+  for (const artifact of logs.artifacts || []) {
+    if (artifact.type === 'TESTSPEC_OUTPUT' && artifact.url) {
+      return await (await fetch(artifact.url)).text();
+    }
+  }
+  return "";
+}
+
+const testSpecOutput = await getTestSpecOutput(arn);
+
+if (!testSpecOutput) {
+  console.error(`Test spec output for run '${arn}' not found`);
+}
+
+console.log(testSpecOutput);
+

scripts/aws-device-farm/s3-client.mjs

@@ -0,0 +1,7 @@
+import { S3Client } from "@aws-sdk/client-s3";
+
+export function getS3Client() {
+  return new S3Client({
+    region: "eu-central-1",
+  });
+}