From f54e43c52a15b3f11b154894873832a811699d2d Mon Sep 17 00:00:00 2001
From: Bart Louwers <bart@emeel.net>
Date: Tue, 10 Sep 2024 22:16:32 +0100
Subject: [PATCH] Attach render test artifacts AWS Device Farm to workflow
 (#2812)

---
 .github/workflows/android-ci.yml              | 17 -----
 .github/workflows/android-device-test.yml     | 16 ++++-
 .../android/app/src/main/AndroidManifest.xml  |  7 +-
 .../aws-device-farm/log-test-spec-output.mjs  | 47 -------------
 .../aws-device-farm/store-test-artifacts.mjs  | 69 +++++++++++++++++++
 5 files changed, 88 insertions(+), 68 deletions(-)
 delete mode 100644 scripts/aws-device-farm/log-test-spec-output.mjs
 create mode 100644 scripts/aws-device-farm/store-test-artifacts.mjs

diff --git a/.github/workflows/android-ci.yml b/.github/workflows/android-ci.yml
index 73cfa5276..5477d63c1 100644
--- a/.github/workflows/android-ci.yml
+++ b/.github/workflows/android-ci.yml
@@ -278,23 +278,6 @@ jobs:
             ./render-test/android/RenderTestsApp-${{ matrix.flavor }}.apk
             ./render-test/android/RenderTests-${{ matrix.flavor }}.apk
 
-      # these two can be (and should be) removed once merged into main
-      - name: Copy render tests and render test app
-        if: matrix.flavor == 'opengl'
-        run: |
-          cp ./render-test/android/RenderTestsApp-opengl.apk ./render-test/android/RenderTestsApp.apk
-          cp ./render-test/android/RenderTests-opengl.apk ./render-test/android/RenderTests.apk
-
-      - name: Store Render Test .apk files
-        if: matrix.flavor == 'opengl'
-        uses: actions/upload-artifact@v4
-        with:
-          name: android-render-tests
-          if-no-files-found: error
-          path: |
-            ./render-test/android/RenderTestsApp.apk
-            ./render-test/android/RenderTests.apk
-
   android-ci-result:
     runs-on: ubuntu-latest
     if: needs.pre_job.outputs.should_skip != 'true' && always()
diff --git a/.github/workflows/android-device-test.yml b/.github/workflows/android-device-test.yml
index 022815c26..177b98c59 100644
--- a/.github/workflows/android-device-test.yml
+++ b/.github/workflows/android-device-test.yml
@@ -29,7 +29,10 @@ jobs:
             appFile: RenderTestsApp-vulkan.apk,
             name: "Android Render Tests (Vulkan)",
             # Google Pixel 7 Pro
-            devicePool: "arn:aws:devicefarm:us-west-2:373521797162:devicepool:20687d72-0e46-403e-8f03-0941850665bc/9692fe7f-86a9-4ecc-908f-175600968564"
+            devicePool: "arn:aws:devicefarm:us-west-2:373521797162:devicepool:20687d72-0e46-403e-8f03-0941850665bc/9692fe7f-86a9-4ecc-908f-175600968564",
+            # android-render-test.yml
+            # see https://github.com/maplibre/ci-runners/tree/main/aws-device-farm/custom-test-envs
+            testSpecArn: "arn:aws:devicefarm:us-west-2:373521797162:upload:20687d72-0e46-403e-8f03-0941850665bc/c1fc7d3e-dfe3-4a31-9ee0-7b0f71b08872"
           },
           {
             artifactName: benchmarkAPKs,
@@ -165,7 +168,16 @@ jobs:
         if: failure() && env.run_device_test == 'true'
         run: |
           npm install
-          node scripts/aws-device-farm/log-test-spec-output.mjs ${{ steps.aws_device_farm_run.outputs.runArn }}
+          temp_dir="$(mktemp -d)"
+          node scripts/aws-device-farm/store-test-artifacts.mjs ${{ steps.aws_device_farm_run.outputs.runArn }} "$temp_dir"
+          zip -r test_artifacts.zip "$temp_dir"
+
+      - name: Upload Test Artifacts
+        if: failure() && env.run_device_test == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-artifacts
+          path: test_artifacts.zip
 
       - name: Generate another token (previous one could have expired)
         if: always()
diff --git a/render-test/android/app/src/main/AndroidManifest.xml b/render-test/android/app/src/main/AndroidManifest.xml
index a601f3ca5..c091e7bf3 100644
--- a/render-test/android/app/src/main/AndroidManifest.xml
+++ b/render-test/android/app/src/main/AndroidManifest.xml
@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
-          android:versionCode="1"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:versionCode="1"
           android:versionName="1.0">
   <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
   <application
@@ -8,7 +9,9 @@
       android:fullBackupContent="false"
       android:icon="@mipmap/ic_launcher"
       android:label="@string/app_name"
-      android:hasCode="true">
+      android:hasCode="true"
+      android:debuggable="true"
+      tools:ignore="HardcodedDebugMode">
 
     <activity android:name="android.app.NativeActivity"
               android:label="@string/app_name"
diff --git a/scripts/aws-device-farm/log-test-spec-output.mjs b/scripts/aws-device-farm/log-test-spec-output.mjs
deleted file mode 100644
index d5e5ab3d1..000000000
--- a/scripts/aws-device-farm/log-test-spec-output.mjs
+++ /dev/null
@@ -1,47 +0,0 @@
-import { ListArtifactsCommand } from "@aws-sdk/client-device-farm";
-import { getDeviceFarmClient } from "./device-farm-client.mjs";
-
-/**
- * @returns {never}
- */
-function usage() {
-  console.error("Logs test spec output from AWS Device Farm");
-  console.error(`Usage: node ${process.argv.at(1)} RUN_ARN`);
-  process.exit(1);
-}
-
-if (process.argv.length !== 3) usage();
-
-const arn = process.argv.at(2);
-
-if (typeof arn !== 'string') usage();
-
-const deviceFarmClient = getDeviceFarmClient();
-
-/**
- * Looks for the run with the provided ARN and returns the test spec output.
- * 
- * @param {string} arn 
- * @returns string
- */
-async function getTestSpecOutput(arn) {
-  const logs = await deviceFarmClient.send(new ListArtifactsCommand({
-    arn,
-    type: 'FILE'
-  }));
-  for (const artifact of logs.artifacts || []) {
-    if (artifact.type === 'TESTSPEC_OUTPUT' && artifact.url) {
-      return await (await fetch(artifact.url)).text();
-    }
-  }
-  return "";
-}
-
-const testSpecOutput = await getTestSpecOutput(arn);
-
-if (!testSpecOutput) {
-  console.error(`Test spec output for run '${arn}' not found`);
-}
-
-console.log(testSpecOutput);
-
diff --git a/scripts/aws-device-farm/store-test-artifacts.mjs b/scripts/aws-device-farm/store-test-artifacts.mjs
new file mode 100644
index 000000000..c02991f15
--- /dev/null
+++ b/scripts/aws-device-farm/store-test-artifacts.mjs
@@ -0,0 +1,69 @@
+import * as fs from "node:fs";
+import { Readable } from "node:stream";
+import { finished } from "node:stream/promises";
+import * as path from "node:path";
+import * as crypto from "node:crypto";
+
+import { ListArtifactsCommand, ListJobsCommand, ListSuitesCommand } from "@aws-sdk/client-device-farm";
+import { getDeviceFarmClient } from "./device-farm-client.mjs";
+
+/**
+ * @returns {never}
+ */
+function usage() {
+  console.error("Stores artifacts from AWS Device Farm run");
+  console.error(`Usage: node ${process.argv.at(1)} RUN_ARN OUTPUT_DIR`);
+  process.exit(1);
+}
+
+if (process.argv.length !== 4) usage();
+
+const arn = process.argv.at(2);
+const outputDirArg = process.argv.at(3);
+
+if (typeof arn !== 'string') usage();
+if (typeof outputDirArg !== 'string') usage();
+
+const outputDir = outputDirArg;
+
+if (!fs.existsSync(outputDir)) {
+  console.error("Output dir does not exist");
+  process.exit(1);
+}
+
+const deviceFarmClient = getDeviceFarmClient();
+
+/**
+ * Looks for the run with the provided ARN and returns the test spec output.
+ * 
+ * @param {string} arn 
+ * @returns string
+ */
+async function getTestSpecOutput(arn) {
+  const jobs = await deviceFarmClient.send(new ListJobsCommand({
+    arn
+  }));
+  
+  await Promise.all((jobs.jobs || []).map(async (job) => {
+    const suites = await deviceFarmClient.send(new ListSuitesCommand({arn: job.arn}));
+    await Promise.all((suites.suites || []).map(async (suite) => {
+      const artifacts = await deviceFarmClient.send(new ListArtifactsCommand({
+        arn: suite.arn,
+        type: 'FILE'
+      }));
+      await Promise.all((artifacts.artifacts || []).map(async (artifact) => {
+        if (!artifact.name || !artifact.url || !artifact.type) return;
+        if (['TESTSPEC_OUTPUT', 'CUSTOMER_ARTIFACT', 'CUSTOMER_ARTIFACT_LOG', 'DEVICE_LOG'].includes(artifact.type)) {
+          const filename = `${artifact.name.replaceAll(' ', '_')}-${crypto.randomBytes(10).toString('hex')}.${artifact.extension}`;
+          const res = await fetch(artifact.url);
+          if (!res.ok || !res.body) return;
+          const destination = path.resolve(outputDir, filename);
+          const fileStream = fs.createWriteStream(destination, { flags: 'wx' });
+          await finished(Readable.fromWeb(/** @type {any} **/ (res.body)).pipe(fileStream));
+        }
+      }));
+    }));
+  }));
+}
+
+await getTestSpecOutput(arn);