tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Store Android benchmark results on device (#2844)

Browse Source
This commit is contained in:
Bart Louwers
2024-09-17 19:14:32 +02:00
committed by GitHub
parent 4f0d5bc2ee
commit 0e0c8da72a
6 changed files with 31 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
.github/workflows/android-device-test.yml vendored
View File

@@ -158,14 +158,8 @@ jobs:
externalData: ${{ env.external_data_arn }}
testSpecArn: ${{ matrix.test.testSpecArn }}
- name: Upload benchmark results to S3
if: always() && env.run_device_test == 'true' && matrix.test.name == 'Android Benchmark'
run: |
npm install
node scripts/aws-device-farm/download-benchmark-results.mjs ${{ steps.aws_device_farm_run.outputs.runArn }}
- name: Log Test Spec Output
if: failure() && env.run_device_test == 'true'
- name: Store Test Artifacts
if: (matrix.test.name == 'Android Benchmark' || failure()) && env.run_device_test == 'true'
run: |
npm install
temp_dir="$(mktemp -d)"
@@ -173,7 +167,7 @@ jobs:
zip -r test_artifacts.zip "$temp_dir"
- name: Upload Test Artifacts
if: failure() && env.run_device_test == 'true'
if: (matrix.test.name == 'Android Benchmark' || failure()) && env.run_device_test == 'true'
uses: actions/upload-artifact@v4
with:
name: "Test Artifacts ${{ matrix.test.name }}"

4
platform/android/MapLibreAndroidTestApp/src/main/AndroidManifest.xml
View File

@@ -13,7 +13,9 @@
android:label="@string/app_name"
android:roundIcon="@drawable/ic_launcher_round"
android:supportsRtl="true"
android:theme="@style/AppTheme">
android:theme="@style/AppTheme"
android:debuggable="true"
tools:ignore="HardcodedDebugMode">
<activity
android:name=".activity.FeatureOverviewActivity"
android:exported="true"

53
platform/android/MapLibreAndroidTestApp/src/main/java/org/maplibre/android/testapp/activity/benchmark/BenchmarkActivity.kt
View File

@@ -18,10 +18,6 @@ import kotlinx.serialization.json.jsonArray
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive
import kotlinx.serialization.json.putJsonObject
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.RequestBody.Companion.toRequestBody
import org.maplibre.android.BuildConfig.GIT_REVISION
import org.maplibre.android.camera.CameraUpdateFactory
import org.maplibre.android.geometry.LatLng
@@ -39,7 +35,6 @@ import java.io.File
data class BenchmarkInputData(
val styleNames: List<String>,
val styleURLs: List<String>,
val resultsAPI: String = ""
) {
init {
if (styleNames.size != styleURLs.size)
@@ -47,10 +42,6 @@ data class BenchmarkInputData(
}
}
/**
* Prepares JSON payload that is sent to the API that collects benchmark results.
* See https://github.com/maplibre/ci-runners
*/
@SuppressLint("NewApi")
fun jsonPayload(styleNames: List<String>, fpsResults: BenchmarkResults, encodingTimeResults: BenchmarkResults, renderingTimeResults: BenchmarkResults): JsonObject {
return buildJsonObject {
@@ -141,14 +132,12 @@ class BenchmarkActivity : AppCompatActivity() {
val jsonElement = Json.parseToJsonElement(jsonFileContents)
val styleNames = jsonElement.jsonObject["styleNames"]?.jsonArray?.map { it.jsonPrimitive.content }
val styleURLs = jsonElement.jsonObject["styleURLs"]?.jsonArray?.map { it.jsonPrimitive.content }
val resultsAPI = jsonElement.jsonObject["resultsAPI"]?.jsonPrimitive?.content
if (styleNames == null || styleURLs == null || resultsAPI == null) {
if (styleNames == null || styleURLs == null) {
throw Error("${jsonFile.name} is missing elements")
}
return BenchmarkInputData(
styleNames = styleNames.toList(),
styleURLs = styleURLs.toList(),
resultsAPI = resultsAPI
styleURLs = styleURLs.toList()
)
} else {
Logger.i(TAG, "${jsonFile.name} not found, reading from developer-config.xml")
@@ -166,8 +155,20 @@ class BenchmarkActivity : AppCompatActivity() {
// return default
return BenchmarkInputData(
styleNames = listOf("MapLibre Demotiles"),
styleURLs = listOf("https://demotiles.maplibre.org/style.json")
styleNames = listOf(
"AWS Open Data Standard Light",
"Facebook Light",
"Americana",
"Protomaps Light",
"Versatiles Colorful"
),
styleURLs = listOf(
"https://maps.geo.us-east-2.amazonaws.com/maps/v0/maps/OpenDataStyle/style-descriptor?key=v1.public.eyJqdGkiOiI1NjY5ZTU4My0yNWQwLTQ5MjctODhkMS03OGUxOTY4Y2RhMzgifR_7GLT66TNRXhZJ4KyJ-GK1TPYD9DaWuc5o6YyVmlikVwMaLvEs_iqkCIydspe_vjmgUVsIQstkGoInXV_nd5CcmqRMMa-_wb66SxDdbeRDvmmkpy2Ow_LX9GJDgL2bbiCws0wupJPFDwWCWFLwpK9ICmzGvNcrPbX5uczOQL0N8V9iUvziA52a1WWkZucIf6MUViFRf3XoFkyAT15Ll0NDypAzY63Bnj8_zS8bOaCvJaQqcXM9lrbTusy8Ftq8cEbbK5aMFapXRjug7qcrzUiQ5sr0g23qdMvnKJQFfo7JuQn8vwAksxrQm6A0ByceEXSfyaBoVpFcTzEclxUomhY.NjAyMWJkZWUtMGMyOS00NmRkLThjZTMtODEyOTkzZTUyMTBi",
"https://external.xx.fbcdn.net/maps/vt/style/canterbury_1_0/?locale=en_US",
"https://americanamap.org/style.json",
"https://api.protomaps.com/styles/v2/light.json?key=e761cc7daedf832a",
"https://tiles.versatiles.org/assets/styles/colorful.json",
)
)
}
@@ -313,28 +314,16 @@ class BenchmarkActivity : AppCompatActivity() {
mapView.onLowMemory()
}
private fun sendResults() {
val api = inputData.resultsAPI
if (api.isEmpty()) {
Logger.i(TAG, "Not sending results to API")
return
}
val client = OkHttpClient()
private fun storeResults() {
val payload = jsonPayload(inputData.styleNames, fpsResults, encodingTimeResults, renderingTimeResults)
Logger.i(TAG, "Sending JSON payload to API: $payload")
val request = Request.Builder()
.url(api)
.post(
Json.encodeToString(payload).toRequestBody("application/json".toMediaType()))
.build()
client.newCall(request).execute()
val dataDir = this.filesDir
val benchmarkResultsFile = File(dataDir, "benchmark_results.json")
benchmarkResultsFile.writeText(Json.encodeToString(payload))
}
private fun benchmarkDone() {
sendResults()
storeResults()
setResult(Activity.RESULT_OK)
finish()
}

1
platform/android/gradle.properties
View File

@@ -4,3 +4,4 @@ systemProp.org.gradle.internal.http.socketTimeout=360000
org.gradle.jvmargs=-Xmx4096M
android.nonTransitiveRClass=false
android.nonFinalResIds=false
android.injected.androidTest.leaveApksInstalledAfterRun=true

4
platform/android/settings.gradle
View File

@@ -8,10 +8,10 @@ rootProject.name = "MapLibre Native for Android"
def renderTestProjectDir = new File(rootDir, '../../render-test/android')
includeBuild(renderTestProjectDir) {
name ="Render Test App"
name ="renderTestApp"
}
def cppTestProjectDir = new File(rootDir, '../../test/android')
includeBuild(cppTestProjectDir) {
name ="C++ Unit Test App"
name ="cppUnitTestsApp"
}

129
scripts/aws-device-farm/download-benchmark-results.mjs
View File

@@ -1,129 +0,0 @@
// This script parses logcat output from AWS Device Farm to extract benchmark results.
// It also uploads the results to S3.
// android-benchmark-render/{gitRevision}/{deviceModel}.json
import https from "node:https";
import readline from "node:readline";
import {
ListJobsCommand,
ListArtifactsCommand,
} from "@aws-sdk/client-device-farm";
import { PutObjectCommand } from "@aws-sdk/client-s3";
import { getDeviceFarmClient } from "./device-farm-client.mjs";
import { getS3Client } from "./s3-client.mjs";
/**
* Retrieves a log file and looks for the line that includes the benchmark results.
* Resolves to the log line or an empty string if the log line was not found.
*
* @param {string} artifactUrl
* @returns {Promise<string>}
*/
function getBenchmarkResult(artifactUrl) {
return new Promise((resolve, reject) => {
https.get(artifactUrl, (res) => {
const rl = readline.createInterface({
input: res,
});
rl.on("line", (line) => {
if (line.includes('Benchmark {"resultsPerStyle"')) {
resolve(line);
rl.close();
}
});
rl.on("close", () => {
resolve("");
});
});
});
}
function usage() {
console.error("Downloads benchmark results from AWS Device Farm");
console.error(`Usage: node ${process.argv.at(1)} RUN_ARN`);
process.exit(1);
}
const deviceFarmClient = getDeviceFarmClient();
if (process.argv.length !== 3) usage();
const arn = process.argv.at(2);
const listJobsCommand = new ListJobsCommand({ arn });
const { jobs } = await deviceFarmClient.send(listJobsCommand);
if (!jobs) throw new Error("Failed to retrieve jobs");
const passedJobs = jobs.filter((job) => job.result === "PASSED" && job.arn);
const failedJobs = jobs.filter((job) => job.result !== "PASSED" || !job.arn);
failedJobs.forEach((job) => {
console.error(
`job.result not PASSED. Skipping.\n$${JSON.stringify(job, null, 0)}`
);
});
// retrieve all artifacts that are files (logcat logs are file artifacts)
const artifacts = await Promise.all(
passedJobs.map((job) =>
deviceFarmClient.send(
new ListArtifactsCommand({
arn: job.arn,
type: "FILE",
})
)
)
);
// only consider artifacts that are logcat logs
const logcatArtifacts = artifacts.flatMap((listArtifactsOutput) =>
listArtifactsOutput.artifacts
? listArtifactsOutput.artifacts.filter(
(artifact) =>
artifact.type === "DEVICE_LOG" && artifact.name === "Logcat"
)
: []
);
const benchmarkResultLogLines = await Promise.all(
logcatArtifacts.flatMap((artifact) => artifact.url ? [getBenchmarkResult(artifact.url)] : [])
);
// parse the log lines to objects
/** @type {{gitRevision?: string, model?: string}[]} */
const benchmarkResults = benchmarkResultLogLines
.map((line) => {
const firstCurly = line.indexOf("{");
if (firstCurly === -1) return "";
return line.substring(firstCurly);
})
.filter((line) => !!line)
.map((jsonVal) => JSON.parse(jsonVal));
const s3Client = getS3Client();
// store benchmark results as JSON files on S3
await Promise.all(
benchmarkResults.flatMap((result) => {
if (typeof result.model !== "string") {
console.error("Result missing model");
return [];
}
if (typeof result.gitRevision !== "string") {
console.error("Result missing gitRevision");
return [];
}
const command = new PutObjectCommand({
Bucket: "maplibre-native", // use environment variable
Key: `android-benchmark-render/${result.gitRevision}/${result.model}.json`,
ContentType: "application/json",
Body: JSON.stringify(result, null, 2),
});
return [s3Client.send(command)];
})
);