Store Android benchmark results on S3 (#2301)

.aspect/rules/external_repository_action_cache/npm_translate_lock_LTE4Nzc1MDcwNjU=

@@ -1,6 +1,7 @@
 # @generated
 # Input hashes for repository rule npm_translate_lock(name = "npm", pnpm_lock = "//:pnpm-lock.yaml").
 # This file should be checked into version control along with the pnpm-lock.yaml file.
-pnpm-lock.yaml=-900989647
-package-lock.json=1930675775
-package.json=965274473
+.npmrc=-592547263
+pnpm-lock.yaml=104874376
+package-lock.json=-583117742
+package.json=-370460401

.github/actions/aws-device-farm-run/action.yml

@@ -43,6 +43,9 @@ inputs:
   testSpecArn:
     description: "ARN of test spec"
     required: false
+outputs:
+  runArn:
+    description: ARN of run
 runs:
   using: "composite"
   steps:
@@ -107,6 +110,7 @@ runs:
           done
 
       - name: Schedule test run
+        id: schedule_run
         shell: bash
         run: |
           arn="$(aws devicefarm schedule-run \
@@ -118,6 +122,8 @@ runs:
             ${{ inputs.externalData && '--configuration extraDataPackageArn=' }}${{ inputs.externalData }} \
             --output text --query "run.arn")"
           
+          echo "runArn=$arn" >> "$GITHUB_OUTPUT"
+          
           # wait until result is not PENDING
           # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/devicefarm/get-run.html#output
           while true; do

.github/workflows/android-device-test.yml

@@ -110,6 +110,7 @@ jobs:
           echo external_data_arn="$upload_arn" >> "$GITHUB_ENV"
 
       - uses: ./.github/actions/aws-device-farm-run
+        id: aws_device_farm_run
         if: env.run_device_test == 'true'
         with:
           name: ${{ matrix.test.name }}
@@ -127,6 +128,11 @@ jobs:
           externalData: ${{ env.external_data_arn }}
           testSpecArn: ${{ matrix.test.testSpecArn }}
 
+      - name: Upload benchmark results to S3
+        run: |
+          npm install --ignore-scripts
+          node scripts/aws-device-farm/download-benchmark-results.mjs ${{ steps.aws_device_farm_run.outputs.runArn }}
+
       - uses: LouisBrunner/checks-action@v2.0.0
         if: always() && (env.run_device_test == 'true' || matrix.test.name == 'Android Render Tests')
         with:

.github/workflows/typecheck-scripts.yml

@@ -0,0 +1,16 @@
+name: typecheck-scripts
+
+on:
+  workflow_dispatch:
+  push:
+  pull_request:
+
+jobs:
+  typecheck-scripts:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - run: |
+          npm install --ignore-scripts
+          npx tsc -p scripts/jsconfig.json

package.json

@@ -27,10 +27,13 @@
     "npm-run-all": "^4.1.5"
   },
   "devDependencies": {
+    "@aws-sdk/client-device-farm": "^3.556.0",
+    "@aws-sdk/client-s3": "^3.556.0",
     "@mapbox/flow-remove-types": "^2.0.0",
     "@mapbox/mvt-fixtures": "3.10.0",
     "@octokit/plugin-retry": "^7.1.0",
     "@octokit/rest": "^20.1.0",
+    "@types/node": "^20.12.7",
     "argparse": "^2.0.1",
     "csscolorparser": "~1.0.3",
     "d3-queue": "3.0.7",
@@ -51,6 +54,7 @@
     "shuffle-seed": "1.1.6",
     "st": "3.0.0",
     "tape": "^5.7.5",
+    "typescript": "^5.4.5",
     "xcode": "^3.0.1"
   },
   "engines": {

scripts/aws-device-farm/download-benchmark-results.mjs

@@ -0,0 +1,130 @@
+// This script parses logcat output from AWS Device Farm to extract benchmark results.
+// It also uploads the results to S3.
+// android-benchmark-render/{gitRevision}/{deviceModel}.json
+
+import https from "node:https";
+import readline from "node:readline";
+
+import {
+  DeviceFarmClient,
+  ListJobsCommand,
+  ListArtifactsCommand,
+} from "@aws-sdk/client-device-farm";
+import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3";
+
+/**
+ * Retrieves a log file and looks for the line that includes the benchmark results.
+ * Resolves to the log line or an empty string if the log line was not found.
+ * 
+ * @param {string} artifactUrl
+ * @returns {Promise<string>}
+ */
+function getBenchmarkResult(artifactUrl) {
+  return new Promise((resolve, reject) => {
+    https.get(artifactUrl, (res) => {
+      const rl = readline.createInterface({
+        input: res,
+      });
+
+      rl.on("line", (line) => {
+        if (line.includes('Benchmark {"resultsPerStyle"')) {
+          resolve(line);
+          rl.close();
+        }
+      });
+
+      rl.on("close", () => {
+        resolve("");
+      });
+    });
+  });
+}
+
+function usage() {
+  console.error("Downloads benchmark results from AWS Device Farm");
+  console.error(`Usage: node ${process.argv.at(1)} RUN_ARN`);
+  process.exit(1);
+}
+
+const deviceFarmClient = new DeviceFarmClient({ region: "us-west-2" });
+
+if (process.argv.length !== 3) usage();
+
+const arn = process.argv.at(2);
+
+const listJobsCommand = new ListJobsCommand({ arn });
+const { jobs } = await deviceFarmClient.send(listJobsCommand);
+
+if (!jobs) throw new Error("Failed to retrieve jobs");
+
+const passedJobs = jobs.filter((job) => job.result === "PASSED" && job.arn);
+const failedJobs = jobs.filter((job) => job.result !== "PASSED" || !job.arn);
+
+failedJobs.forEach((job) => {
+  console.error(
+    `job.result not PASSED. Skipping.\n$${JSON.stringify(job, null, 0)}`
+  );
+});
+
+// retrieve all artifacts that are files (logcat logs are file artifacts)
+const artifacts = await Promise.all(
+  passedJobs.map((job) =>
+    deviceFarmClient.send(
+      new ListArtifactsCommand({
+        arn: job.arn,
+        type: "FILE",
+      })
+    )
+  )
+);
+
+// only consider artifacts that are logcat logs
+const logcatArtifacts = artifacts.flatMap((listArtifactsOutput) =>
+  listArtifactsOutput.artifacts
+    ? listArtifactsOutput.artifacts.filter(
+        (artifact) =>
+          artifact.type === "DEVICE_LOG" && artifact.name === "Logcat"
+      )
+    : []
+);
+
+const benchmarkResultLogLines = await Promise.all(
+  logcatArtifacts.flatMap((artifact) => artifact.url ? [getBenchmarkResult(artifact.url)] : [])
+);
+
+// parse the log lines to objects
+/** @type {{gitRevision?: string, model?: string}[]} */
+const benchmarkResults = benchmarkResultLogLines
+  .map((line) => {
+    const firstCurly = line.indexOf("{");
+    if (firstCurly === -1) return "";
+    return line.substring(firstCurly);
+  })
+  .filter((line) => !!line)
+  .map((jsonVal) => JSON.parse(jsonVal));
+
+const s3Client = new S3Client({
+  region: "eu-central-1",
+});
+
+// store benchmark results as JSON files on S3
+await Promise.all(
+  benchmarkResults.flatMap((result) => {
+    if (typeof result.model !== "string") {
+      console.error("Result missing model");
+      return [];
+    }
+    if (typeof result.gitRevision !== "string") {
+      console.error("Result missing gitRevision");
+      return [];
+    }
+
+    const command = new PutObjectCommand({
+      Bucket: "maplibre-native", // use environment variable
+      Key: `android-benchmark-render/${result.gitRevision}/${result.model}.json`,
+      ContentType: "application/json",
+      Body: JSON.stringify(result, null, 2),
+    });
+    return [s3Client.send(command)];
+  })
+);

scripts/jsconfig.json

@@ -0,0 +1,11 @@
+{
+  "compilerOptions": {
+    "strict": true,
+    "allowJs": true,
+    "checkJs": true,
+    "module": "NodeNext"
+  },
+  "include": [
+    "**/*.mjs"
+  ]
+}