Automate Android benchmark running / data processing (#2991)

2024-11-11 16:57:26 +01:00
parent ed353f24bf
commit 6b12d58a26
18 changed files with 723 additions and 304 deletions
--- a/scripts/aws-device-farm/aws-device-farm-run.sh
+++ b/scripts/aws-device-farm/aws-device-farm-run.sh
@@ -0,0 +1,106 @@
+#!/bin/bash
+
+# List of required environment variables
+required_vars=(
+  "AWS_DEVICE_FARM_PROJECT_ARN"
+  "AWS_DEVICE_FARM_DEVICE_POOL_ARN"
+  "appFile"
+  "appType"
+  "testFile"
+  "testType"
+  "testPackageType"
+  "name"
+)
+
+check_var() {
+  var_name=$1
+  if [[ -z "${!var_name}" ]]; then
+      echo "Error: Environment variable $var_name is not set."
+      exit 1
+  fi
+}
+
+for var in "${required_vars[@]}"; do
+  check_var "$var"
+done
+
+if [[ ! -f "$appFile" ]]; then
+  echo "Error: App file $appFile does not exist."
+  exit 1
+fi
+
+if [[ ! -f "$testFile" ]]; then
+  echo "Error: Test file $testFile does not exist."
+  exit 1
+fi
+
+# Create upload app
+response=$(aws devicefarm create-upload --type "$appType" --name "$(basename "$appFile")" --project-arn "$AWS_DEVICE_FARM_PROJECT_ARN")
+echo "$response" >&2
+app_arn="$(jq -r '.upload.arn' <<< "$response")"
+app_url="$(jq -r '.upload.url' <<< "$response")"
+
+# Create upload test package
+response=$(aws devicefarm create-upload --type "$testPackageType" --name "$(basename "$testFile")" --project-arn "$AWS_DEVICE_FARM_PROJECT_ARN")
+echo "$response" >&2
+test_package_arn="$(jq -r '.upload.arn' <<< "$response")"
+test_package_url="$(jq -r '.upload.url' <<< "$response")"
+
+# Upload app and test package
+curl -T "$appFile" "$app_url"
+curl -T "$testFile" "$test_package_url"
+
+max_checks=10
+sleep_time=5
+
+check_status() {
+  aws devicefarm get-upload --arn "$1" | jq -r '.upload.status'
+}
+
+while ((max_checks--)); do
+  status_app="$(check_status "$app_arn")"
+  status_test_package="$(check_status "$test_package_arn")"
+
+  status_app="$status_app"
+  status_test_package="$status_test_package"
+
+  if [[ "$status_app" == "SUCCEEDED" && "$status_test_package" == "SUCCEEDED" ]]; then
+    echo "Uploads succeeded"  >&2
+    break
+  elif ((max_checks == 0)); then
+    echo "App or test package failed to upload"  >&2
+    exit 1
+  fi
+
+  sleep $sleep_time
+done
+
+# Schedule test run
+arn="$(aws devicefarm schedule-run \
+  --project-arn "$AWS_DEVICE_FARM_PROJECT_ARN" \
+  --name "MapLibre Native $name" \
+  --app-arn "$app_arn" \
+  --device-pool-arn "$AWS_DEVICE_FARM_DEVICE_POOL_ARN" \
+  --test type=$testType,testPackageArn=$test_package_arn${testFilter:+,filter=$testFilter}${testSpecArn:+,testSpecArn=$testSpecArn} \
+  --execution-configuration videoCapture=false \
+  --output text --query run.arn)"
+
+echo "$arn"
+
+if [[ "$wait_for_completion" != "true" ]]; then
+  echo "Not waiting for run to complete" >&2
+  exit 0
+fi
+
+# wait until result is not PENDING
+# https://awscli.amazonaws.com/v2/documentation/api/latest/reference/devicefarm/get-run.html#output
+while true; do
+  sleep 30
+  result="$(aws devicefarm get-run --arn "$arn" --output text --query "run.result")"
+  case $result in
+    FAILED|ERRORED|STOPPED) echo "Run $result" && exit 1 ;;
+    SKIPPED|PASSED) echo "Run $result" && exit 0 ;;
+    PENDING) continue ;;
+    *) echo "Unexpected run result $result" && exit 1 ;;
+  esac
+done
--- a/scripts/aws-device-farm/collect-benchmark-outputs.mjs
+++ b/scripts/aws-device-farm/collect-benchmark-outputs.mjs
@@ -0,0 +1,87 @@
+import * as fs from "node:fs/promises";
+import { exec } from 'node:child_process';
+import { parseArgs, promisify } from "node:util";
+import * as path from "node:path";
+
+const execPromise = promisify(exec);
+
+/**
+ * @returns {never}
+ */
+function usage() {
+  console.error("Collects benchmark outputs from AWS Device Farm artifacts");
+  console.error("Usage: node collect-benchmark-outputs.mjs --inputDir INPUT_DIR --outputDir OUTPUT_DIR");
+  process.exit(1);
+}
+
+function getArgs() {
+  const {
+    values
+  } = parseArgs({
+    options: {
+      outputDir: {
+        type: "string",
+      },
+      inputDir: {
+        type: "string"
+      },
+    },
+  });
+  const { outputDir, inputDir } = values;
+  if (typeof outputDir !== 'string') usage();
+
+  if (typeof inputDir !== 'string') usage();
+
+  return {
+    outputDir,
+    inputDir
+  }
+}
+
+const { outputDir, inputDir } = getArgs();
+
+await unzipFilesInDirectory(inputDir);
+await writeResultsToOutputDir();
+
+///////////////////////////////////////////////////////////////////////////////
+
+/**
+ * 
+ * @param {string} directory 
+ */
+async function unzipFilesInDirectory(directory) {
+  try {
+    const files = await fs.readdir(directory);
+    const zipFiles = files.filter(file => file.endsWith('.zip'));
+
+    for (const zipFile of zipFiles) {
+      const zipFilePath = path.join(directory, zipFile);
+      const { name } = path.parse(zipFile);
+      const extractDir = path.join(directory, name);
+
+      await fs.mkdir(extractDir, { recursive: true });
+
+      await execPromise(`unzip "${zipFilePath}" -d "${extractDir}"`);
+      console.log(`Extracted ${zipFile} to ${extractDir}`);
+    }
+
+  } catch (error) {
+    if (error instanceof Error) console.error(`Error unzipping files: ${error.message}`);
+  }
+}
+
+async function writeResultsToOutputDir() {
+  for await (const benchmarkResultPath of fs.glob("**/benchmark_results.json", {cwd: inputDir})) {
+    try {
+      const fullBenchmarkResultPath  = path.join(inputDir, benchmarkResultPath);
+      const benchmarkResultContents = await fs.readFile(fullBenchmarkResultPath, {encoding: 'utf-8'});
+      const benchmarkResults = JSON.parse(benchmarkResultContents);
+      const { timestamp } = benchmarkResults;
+      if (typeof timestamp !== 'number') throw new Error("No timestamp found in bechmark result");
+      await fs.copyFile(fullBenchmarkResultPath, path.join(outputDir, `${timestamp}.json`));
+    } catch (err) {
+      console.error(`Error ${err}. Skipping: '${benchmarkResultPath}'`);
+    }
+  }
+  console.log(`Wrote benchmark results to ${outputDir}`);
+}
--- a/scripts/aws-device-farm/device-farm-client.mjs
+++ b/scripts/aws-device-farm/device-farm-client.mjs
@@ -3,5 +3,5 @@ import {
 } from "@aws-sdk/client-device-farm";

 export function getDeviceFarmClient() {
-  return new DeviceFarmClient({ region: "us-west-2" });
+  return new DeviceFarmClient({ region: "us-west-2", retryMode: "adaptive", maxAttempts: 10 });
 }
--- a/scripts/aws-device-farm/plot-android-benchmark-results.py
+++ b/scripts/aws-device-farm/plot-android-benchmark-results.py
@@ -0,0 +1,98 @@
+import sqlite3
+import matplotlib.pyplot as plt
+import numpy as np
+import math
+import argparse
+
+parser = argparse.ArgumentParser("plot-android-benchmark-results")
+parser.add_argument("db", help="Path to SQLite database with results", type=str)
+args = parser.parse_args()
+
+# Connect to the SQLite database
+conn = sqlite3.connect(args.db)  # Replace with your database file path
+cursor = conn.cursor()
+
+# Query to get the average FPS and variance for each style, renderer, and model where syncRendering=0
+query = """
+SELECT deviceManufacturer, model, styleName, renderer,
+       AVG(fps) AS avg_fps,
+       (
+           AVG(fps * fps) - AVG(fps) * AVG(fps)
+       ) AS variance_fps
+FROM benchmark_result
+WHERE syncRendering = 0
+GROUP BY deviceManufacturer, model, styleName, renderer
+ORDER BY deviceManufacturer, model, styleName, renderer
+"""
+
+# Execute the query
+cursor.execute(query)
+rows = cursor.fetchall()
+
+# Close the connection to the database
+conn.close()
+
+# Organize the data into a dictionary for plotting
+data = {}
+for row in rows:
+    manufacturer, model, style, renderer, avg_fps, variance_fps = row
+    device = f"{manufacturer} {model}"
+    if device not in data:
+        data[device] = {}
+    if style not in data[device]:
+        data[device][style] = {'fps': [], 'stddev': []}
+    stddev_fps = math.sqrt(max(variance_fps, 0))  # Calculate the standard deviation and ensure it's not negative
+    data[device][style]['fps'].append((renderer, avg_fps))
+    data[device][style]['stddev'].append((renderer, stddev_fps))
+
+# Define the colors for each renderer
+colors = {'legacy': 'gray', 'drawable': 'blue', 'vulkan': 'red'}
+renderer_order = ['legacy', 'drawable', 'vulkan']
+
+legend_names = {'legacy': 'Legacy', 'drawable': 'OpenGL', 'vulkan': 'Vulkan'}
+
+# Create subplots for each device
+num_devices = len(data)
+fig, axes = plt.subplots(num_devices, 1, figsize=(10, 6 * num_devices))
+
+if num_devices == 1:
+    axes = [axes]
+
+# Plot data for each device
+for ax, (device, styles) in zip(axes, data.items()):
+    # Get a sorted list of styles
+    style_names = sorted(styles.keys())
+
+    # Number of styles
+    n_styles = len(style_names)
+
+    # Create an array with the position of each group of bars
+    bar_width = 0.25
+    index = np.arange(n_styles)
+
+    # Plot each renderer's bar with error bars in the defined order
+    for i, renderer in enumerate(renderer_order):
+        avg_fps_values = []
+        stddev_values = []
+        for style in style_names:
+            fps_values = dict(styles[style]['fps'])
+            stddev_values_dict = dict(styles[style]['stddev'])
+            # Use get to return a default of 0 if the renderer is not present
+            avg_fps_values.append(fps_values.get(renderer, 0))
+            stddev_values.append(stddev_values_dict.get(renderer, 0))
+        ax.bar(index + i * bar_width, avg_fps_values, bar_width, yerr=stddev_values,
+               label=legend_names[renderer], color=colors[renderer], capsize=5, error_kw={'elinewidth': 1, 'ecolor': 'black'})
+
+    # Set the title, labels, ticks, etc.
+    ax.set_title(device)
+    ax.set_xlabel('Style')
+    ax.set_ylabel('Average FPS')
+    ax.set_xticks(index + bar_width)
+    ax.set_xticklabels(style_names)
+    ax.legend(title='Renderer')
+
+# Adjust layout
+plt.tight_layout()
+
+# Show the plot
+plt.savefig('plot.png')
--- a/scripts/aws-device-farm/store-test-artifacts.mjs
+++ b/scripts/aws-device-farm/store-test-artifacts.mjs
@@ -8,6 +8,18 @@ import { parseArgs } from "node:util";
 import { ArtifactType, ListArtifactsCommand, ListJobsCommand, ListSuitesCommand } from "@aws-sdk/client-device-farm";
 import { getDeviceFarmClient } from "./device-farm-client.mjs";

+/**
+ * @returns {never}
+ */
+function usage() {
+  console.error("Stores artifacts from AWS Device Farm run");
+  console.error("Usage: node store-test-artifacts.mjs --outputDir OUTPUT_DIR --runArn RUN_ARN");
+  console.error("Arguments:")
+  console.error("--customerArtifacts: only download customer artifacts");
+  console.error("--testsSuite: only download stuff from Tests Suite");
+  process.exit(1);
+}
+
 function getArgs() {
  const {
    values
@@ -17,10 +29,11 @@ function getArgs() {
        type: "string",
      },
      runArn: {
-        type: "string"
+        type: "string",
+        multiple: true
      },
      testsSuite: {
-       type: "boolean" 
+        type: "boolean"
      },
      customerArtifacts: {
        type: "boolean"
@@ -30,7 +43,7 @@ function getArgs() {
  const { outputDir, runArn } = values;
  if (typeof outputDir !== 'string') usage();

-  if (typeof runArn !== 'string') usage();
+  if (!runArn || !runArn.length) usage();

  function suitesFilter() {
    const names = new Set();
@@ -57,56 +70,54 @@ function getArgs() {

 const { outputDir, runArn, suitesFilter, artifactsToDownload } = getArgs();

-/**
- * @returns {never}
- */
-function usage() {
-  console.error("Stores artifacts from AWS Device Farm run");
-  console.error("Usage: node store-test-artifacts.mjs --outputDir OUTPUT_DIR --runArn RUN_ARN");
-  console.error("Arguments:")
-  console.error("--customerArtifacts: only download customer artifacts");
-  console.error("--testsSuite: only download stuff from Tests Suite");
-  process.exit(1);
-}
-
 if (!fs.existsSync(outputDir)) {
  console.error("Output dir does not exist");
  process.exit(1);
 }

 const deviceFarmClient = getDeviceFarmClient();
+await storeRunArtifacts(runArn, outputDir);
+
+///////////////////////////////////////////////////////////////////////////////

 /**
 * Looks for the run with the provided ARN and returns the test spec output.
 * 
- * @param {string} arn 
+ * @param {string[]} arnArr
+ * @param {string} outputDir
 * @returns string
 */
-async function getTestSpecOutput(arn) {
-  const jobs = await deviceFarmClient.send(new ListJobsCommand({
-    arn
-  }));
-  
-  await Promise.all((jobs.jobs || []).map(async (job) => {
-    const suites = await deviceFarmClient.send(new ListSuitesCommand({arn: job.arn}));
-    await Promise.all((suites.suites || []).filter(suitesFilter).map(async (suite) => {
-      const artifacts = await deviceFarmClient.send(new ListArtifactsCommand({
-        arn: suite.arn,
-        type: 'FILE'
-      }));
-      await Promise.all((artifacts.artifacts || []).map(async (artifact) => {
-        if (!artifact.name || !artifact.url || !artifact.type) return;
-        if (artifactsToDownload.includes(artifact.type)) {
-          const filename = `${artifact.name.replaceAll(' ', '_')}-${crypto.randomBytes(10).toString('hex')}.${artifact.extension}`;
-          const res = await fetch(artifact.url);
-          if (!res.ok || !res.body) return;
-          const destination = path.resolve(outputDir, filename);
-          const fileStream = fs.createWriteStream(destination, { flags: 'wx' });
-          await finished(Readable.fromWeb(/** @type {any} **/ (res.body)).pipe(fileStream));
-        }
+async function storeRunArtifacts(arnArr, outputDir) {
+  for (const arn of arnArr) {
+    const jobs = await deviceFarmClient.send(new ListJobsCommand({
+      arn
+    }));
+
+    await Promise.all((jobs.jobs || []).map(async (job) => {
+      const suites = await deviceFarmClient.send(new ListSuitesCommand({ arn: job.arn }));
+      await Promise.all((suites.suites || []).filter(suitesFilter).map(async (suite) => {
+        const artifacts = await deviceFarmClient.send(new ListArtifactsCommand({
+          arn: suite.arn,
+          type: 'FILE'
+        }));
+        await Promise.all((artifacts.artifacts || []).map(async (artifact) => {
+          if (!artifact.name || !artifact.url || !artifact.type) return;
+          if (artifactsToDownload.includes(artifact.type)) {
+            if (!artifact.arn) return;
+            const destination = path.join(outputDir, `${Buffer.from(artifact.arn).toString('base64')}.${artifact.extension}`);
+            try {
+              await fs.promises.access(destination);
+              return; // already exists
+            } catch (err) {
+            }
+            const res = await fetch(artifact.url);
+            if (!res.ok || !res.body) return;
+            const fileStream = fs.createWriteStream(destination, { flags: 'wx' });
+            await finished(Readable.fromWeb(/** @type {any} **/(res.body)).pipe(fileStream));
+          }
+        }));
      }));
    }));
-  }));
+  }
+  console.log(`Wrote run artifacts to ${outputDir}`)
 }
-
-await getTestSpecOutput(runArn);
--- a/scripts/aws-device-farm/update-benchmark-db.mjs
+++ b/scripts/aws-device-farm/update-benchmark-db.mjs
@@ -0,0 +1,195 @@
+import { parseArgs, promisify } from "node:util";
+import * as path from "node:path";
+import os, { type } from "node:os";
+import * as fs from "node:fs/promises";
+import { exec as execCallback } from "node:child_process";
+import { DatabaseSync } from "node:sqlite";
+
+const exec = promisify(execCallback);
+
+/**
+ * @returns {never}
+ */
+function usage() {
+  console.error("Update database with benchmark results");
+  console.error("Usage: node update-benchmark-db.mjs --dbPath PATH [--gitRevision SHA] [--jsonDir JSON_DIR] [--download]");
+  process.exit(1);
+}
+
+function getArgs() {
+  const {
+    values
+  } = parseArgs({
+    options: {
+      dbPath: {
+        type: "string"
+      },
+      gitRevision: {
+        type: "string",
+      },
+      jsonDir: {
+        type: "string"
+      },
+      download: {
+        type: "boolean"
+      }
+    },
+  });
+  const { gitRevision, jsonDir, download, dbPath } = values;
+
+  if (typeof dbPath !== 'string') {
+    console.error("--dbPath is required");
+    throw usage();
+  }
+
+  return {
+    gitRevision,
+    jsonDir,
+    download,
+    dbPath
+  }
+}
+
+const { gitRevision, jsonDir, download, dbPath } = getArgs();
+
+
+async function getDir() {
+  if (jsonDir) {
+    return jsonDir;
+  }
+  if (!download) {
+    console.error("--jsonDir is required when not using --download");
+    throw usage();
+  }
+  return await fs.mkdtemp(path.join(os.tmpdir(), `benchmark-results-${gitRevision}-`));
+}
+
+let dir = await getDir();
+
+if (download) await downloadBenchmarkResults(dir);
+
+const results = await loadResults(dir);
+
+updateDb(dbPath, results);
+
+///////////////////////////////////////////////////////////////////////////////
+
+/**
+ * @param {string | undefined} gitRevision
+ * @returns {Promise<string>}
+ */
+async function downloadBenchmarkResults(gitRevision) {
+  if (!gitRevision) {
+    console.error("--gitRevision is required when downloading");
+    throw usage();
+  }
+
+  const bucket = "maplibre-native";
+  const prefix = `android-benchmark-render/${gitRevision}`;
+
+  try {
+    // Run the AWS CLI sync command
+    const { stdout, stderr } = await exec(`aws s3 sync s3://${bucket}/${prefix} ${dir}`);
+    console.log(stdout);
+    if (stderr) {
+      console.error(`Warning: ${stderr}`);
+    }
+  } catch (error) {
+    throw new Error(`Failed to sync directory s3://${bucket}/${prefix}. ${error}`);
+  }
+
+  console.log(`Downloaded benchmark results to ${dir}`);
+  return dir;
+}
+
+/**
+ * @param {string} dir
+ * @returns {Promise<Object[]>} An array with parsed JSON objects from each file
+ */
+async function loadResults(dir) {
+  return (await Promise.all((await fs.readdir(dir))
+    .map(filePath => path.join(dir, filePath))
+    .filter(filePath => path.extname(filePath) === ".json")
+    .map(filePath => fs.readFile(filePath, "utf-8"))))
+    .map(fileContents => JSON.parse(fileContents));
+}
+
+/**
+ * @param {string} dbPath
+ * @param {any[]} results 
+ */
+function updateDb(dbPath, results) {
+  const db = new DatabaseSync(dbPath);
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS benchmark_result (
+      id TEXT PRIMARY KEY,
+      styleName TEXT NOT NULL,
+      fps REAL NOT NULL,
+      avgEncodingTime REAL NOT NULL,
+      low1pEncodingTime REAL NOT NULL,
+      avgRenderingTime REAL NOT NULL,
+      low1pRenderingTime REAL NOT NULL,
+      syncRendering BOOLEAN NOT NULL,
+      deviceManufacturer TEXT NOT NULL,
+      model TEXT NOT NULL,
+      renderer TEXT NOT NULL,
+      gitRevision TEXT NOT NULL
+    )`);
+
+  const deleteExisting = db.prepare(
+    `DELETE FROM benchmark_result WHERE id = ?`);
+
+  const stmt = db.prepare(
+    `INSERT INTO benchmark_result (
+      id,
+      styleName,
+      fps,
+      avgEncodingTime,
+      low1pEncodingTime,
+      avgRenderingTime,
+      low1pRenderingTime,
+      syncRendering,
+      deviceManufacturer,
+      model,
+      renderer,
+      gitRevision
+    ) VALUES (
+      @id,
+      @styleName,
+      @fps,
+      @avgEncodingTime,
+      @low1pEncodingTime,
+      @avgRenderingTime,
+      @low1pRenderingTime,
+      @syncRendering,
+      @deviceManufacturer,
+      @model,
+      @renderer,
+      @gitRevision
+  ) RETURNING *`);
+
+  for (const run of results) {
+    for (const [index, result] of run.results.entries()) {
+      const id = `${run.timestamp}-${index}`;
+      const { 
+        deviceManufacturer,
+        gitRevision,
+        model,
+        renderer,
+      } = run;
+      deleteExisting.run(id);
+      const params = {
+        ...result,
+        id,
+        deviceManufacturer,
+        gitRevision,
+        model,
+        renderer,
+        syncRendering: result.syncRendering ? 1 : 0,
+      };
+      stmt.run(params);
+    }
+  }
+
+  return db;
+}
--- a/scripts/aws-device-farm/upload-benchmark-outputs-to-s3.mjs
+++ b/scripts/aws-device-farm/upload-benchmark-outputs-to-s3.mjs
@@ -0,0 +1,87 @@
+import { parseArgs } from "node:util";
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3";
+
+/**
+ * @returns {never}
+ */
+function usage() {
+  console.error("Uploads a directory of benchmark results (JSON files) to S3");
+  console.error("Usage: node upload-benchmark-outputs-to-s3.mjs --dir DIR");
+  process.exit(1);
+}
+
+function getArgs() {
+  const {
+    values
+  } = parseArgs({
+    options: {
+      dir: {
+        type: "string",
+      },
+    },
+  });
+  const { dir } = values;
+  if (typeof dir !== 'string') usage();
+
+  return {
+    dir
+  }
+}
+
+const { dir } = getArgs();
+
+processFiles(dir);
+
+///////////////////////////////////////////////////////////////////////////////
+
+/**
+ * 
+ * @param {string} filePath 
+ * @param {string} gitRevision 
+ * @param {string} filename 
+ */
+async function uploadFileToS3(filePath, gitRevision, filename) {
+  const s3Client = new S3Client({ region: "eu-central-1" });
+  const bucket = "maplibre-native";
+  const key = `android-benchmark-render/${gitRevision}/${filename}`;
+
+  try {
+    const fileContent = await fs.readFile(filePath);
+    const command = new PutObjectCommand({
+      Bucket: bucket,
+      Key: key,
+      Body: fileContent,
+    });
+    await s3Client.send(command);
+    console.log(`Uploaded ${filename} to s3://${bucket}/${key}`);
+  } catch (error) {
+    console.error(`Failed to upload ${filename}:`, error);
+  }
+}
+
+/**
+ * 
+ * @param {string} dir 
+ */
+async function processFiles(dir) {
+  const files = await fs.readdir(dir);
+  await Promise.all(files.map(async (filename) => {
+    const filePath = path.join(dir, filename);
+    if (path.extname(filename) !== ".json") return;
+
+    try {
+      const fileContent = await fs.readFile(filePath, "utf-8");
+      const data = JSON.parse(fileContent);
+      if (!data.gitRevision) {
+        console.error(`File ${filename} does not have a 'gitRevision' key. Skipping.`);
+        return;
+      }
+
+      await uploadFileToS3(filePath, data.gitRevision, filename);
+    } catch (error) {
+      console.error(`Error processing file ${filename}:`, error);
+    }
+  }));
+}