Automate Android benchmark running / data processing (#2991)
This commit is contained in:
Bart Louwers
60
.github/workflows/android-device-test.yml
vendored
60
.github/workflows/android-device-test.yml
vendored
@@ -70,6 +70,10 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: '.nvmrc'
|
||||
|
||||
- id: parent_workflow
|
||||
run: |
|
||||
conclusion=$(curl ${{ github.event.workflow_run.jobs_url }} | jq -r '.jobs[] | select(.name == "android-build").conclusion')
|
||||
@@ -130,33 +134,25 @@ jobs:
|
||||
aws-region: us-west-2
|
||||
role-to-assume: ${{ vars.OIDC_AWS_ROLE_TO_ASSUME }}
|
||||
role-session-name: ${{ github.run_id }}
|
||||
|
||||
- name: Upload external data
|
||||
if: env.run_device_test == 'true'
|
||||
|
||||
- name: Run ${{ matrix.test.name }} on AWS Device Farm
|
||||
run: |
|
||||
export RESULTS_API=${{ secrets.MLN_RESULTS_API }}
|
||||
export AWS_DEVICE_FARM_PROJECT_ARN=${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }}
|
||||
upload_arn="$(.github/workflows/android-device-test/upload-external-data.sh)"
|
||||
echo external_data_arn="$upload_arn" >> "$GITHUB_ENV"
|
||||
export name="${{ matrix.test.name }}"
|
||||
export appType=ANDROID_APP
|
||||
export appFile="${{ matrix.test.appFile }}"
|
||||
export testFile="${{ matrix.test.testFile }}"
|
||||
export testPackageType=INSTRUMENTATION_TEST_PACKAGE
|
||||
export testType=INSTRUMENTATION
|
||||
export testFilter="${{ matrix.test.testFilter }}"
|
||||
export AWS_ACCESS_KEY_ID="${{ secrets.AWS_ACCESS_KEY_ID }}"
|
||||
export AWS_SECRET_ACCESS_KEY="${{ secrets.AWS_SECRET_ACCESS_KEY }}"
|
||||
export AWS_ROLE_TO_ASSUME="${{ vars.OIDC_AWS_ROLE_TO_ASSUME }}"
|
||||
export AWS_DEVICE_FARM_PROJECT_ARN="${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }}"
|
||||
export AWS_DEVICE_FARM_DEVICE_POOL_ARN="${{ matrix.test.devicePool }}"
|
||||
export testSpecArn="${{ matrix.test.testSpecArn }}"
|
||||
export wait_for_completion=true
|
||||
|
||||
- uses: ./.github/actions/aws-device-farm-run
|
||||
id: aws_device_farm_run
|
||||
if: env.run_device_test == 'true'
|
||||
with:
|
||||
name: ${{ matrix.test.name }}
|
||||
appType: ANDROID_APP
|
||||
appFile: ${{ matrix.test.appFile }}
|
||||
testFile: ${{ matrix.test.testFile }}
|
||||
testPackageType: INSTRUMENTATION_TEST_PACKAGE
|
||||
testType: INSTRUMENTATION
|
||||
testFilter: ${{ matrix.test.testFilter }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_ROLE_TO_ASSUME: ${{ vars.OIDC_AWS_ROLE_TO_ASSUME }}
|
||||
AWS_DEVICE_FARM_PROJECT_ARN: ${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }}
|
||||
AWS_DEVICE_FARM_DEVICE_POOL_ARN: ${{ matrix.test.devicePool }}
|
||||
externalData: ${{ env.external_data_arn }}
|
||||
testSpecArn: ${{ matrix.test.testSpecArn }}
|
||||
echo run_arn="$(./scripts/aws-device-farm/aws-device-farm-run.sh)" > "$GITHUB_ENV"
|
||||
|
||||
- name: Store Test Artifacts
|
||||
if: (matrix.test.name == 'Android Benchmark' || failure()) && env.run_device_test == 'true'
|
||||
@@ -164,19 +160,15 @@ jobs:
|
||||
npm install
|
||||
results_dir="$(mktemp -d)"
|
||||
echo results_dir="$results_dir" >> "$GITHUB_ENV"
|
||||
node scripts/aws-device-farm/store-test-artifacts.mjs --runArn ${{ steps.aws_device_farm_run.outputs.runArn }} --outputDir "$results_dir"
|
||||
zip -r test_artifacts.zip ${{ env.results_dir }}
|
||||
node scripts/aws-device-farm/store-test-artifacts.mjs --runArn ${{ env.run_arn }} --outputDir "$results_dir"
|
||||
zip -r test_artifacts.zip "$results_dir"
|
||||
|
||||
- name: Store Benchmark Results
|
||||
if: matrix.test.name == 'Android Benchmark' && env.run_device_test == 'true'
|
||||
run: |
|
||||
for zipfile in ${{ env.results_dir }}/*.zip; do
|
||||
unzip "$zipfile" -d "${zipfile%.zip}"
|
||||
done
|
||||
|
||||
find "${{ env.results_dir }}" -name 'benchmark_results.json' | while read -r benchmark_json; do
|
||||
aws s3 cp "$benchmark_json" "s3://maplibre-native/android-benchmark-render/$(uuidgen).json"
|
||||
done
|
||||
benchmark_results_dir="$(mktemp -d)"
|
||||
node scripts/aws-device-farm/collect-benchmark-outputs.mjs --inputDir "${{ env.results_dir }}" --outputDir "$benchmark_results_dir"
|
||||
node scripts/aws-device-farm/upload-benchmark-outputs-to-s3.mjs --dir "$benchmark_results_dir"
|
||||
|
||||
- name: Upload Test Artifacts
|
||||
if: (matrix.test.name == 'Android Benchmark' || failure()) && env.run_device_test == 'true'
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
# Check if the required variables are defined
|
||||
if [ -z "$RESULTS_API" ] || [ -z "$AWS_DEVICE_FARM_PROJECT_ARN" ]; then
|
||||
echo "Error: Missing required variables."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cd "$(mktemp -d)"
|
||||
|
||||
cat << EOF > instrumentation-test-input.json
|
||||
{
|
||||
"styleNames": ["Facebook Light", "MapTiler Basic", "Americana"],
|
||||
"styleURLs": [
|
||||
"https://external.xx.fbcdn.net/maps/vt/style/canterbury_1_0/?locale=en_US",
|
||||
"https://api.maptiler.com/maps/basic-v2/style.json?key=get_your_own_OpIi9ZULNHzrESv6T2vL",
|
||||
"https://americanamap.org/style.json"
|
||||
],
|
||||
"resultsAPI": "$RESULTS_API",
|
||||
"apiKey": "get_your_own_OpIi9ZULNHzrESv6T2vL"
|
||||
}
|
||||
EOF
|
||||
|
||||
>&2 zip instrumentation-test-input.zip instrumentation-test-input.json
|
||||
upload_json="$(aws devicefarm create-upload --project-arn "$AWS_DEVICE_FARM_PROJECT_ARN" --type EXTERNAL_DATA --name instrumentation-test-input.zip --output json)"
|
||||
upload_url="$(echo "$upload_json" | jq -r '.upload.url')"
|
||||
upload_arn="$(echo "$upload_json" | jq -r '.upload.arn')"
|
||||
curl -T instrumentation-test-input.zip "$upload_url"
|
||||
|
||||
retries=5
|
||||
while true; do
|
||||
upload_status="$(aws devicefarm get-upload --arn "$upload_arn" --output text --query upload.status)"
|
||||
>&2 echo "Upload $upload_status"
|
||||
sleep 1
|
||||
if [[ "$upload_status" == "SUCCEEDED" ]]; then
|
||||
break
|
||||
fi
|
||||
|
||||
if (( --retries == 0 )); then
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "$upload_arn"
|
||||
29
.github/workflows/ios-device-test.yml
vendored
29
.github/workflows/ios-device-test.yml
vendored
@@ -49,20 +49,23 @@ jobs:
|
||||
with:
|
||||
files: "${{ matrix.test.xcTestFile }}, ${{ matrix.test.ipaFile }}"
|
||||
|
||||
- uses: ./.github/actions/aws-device-farm-run
|
||||
- name: Run ${{ matrix.test.name }} on AWS Device Farm
|
||||
if: steps.check_files.outputs.files_exists == 'true'
|
||||
with:
|
||||
name: ${{ matrix.test.name }}
|
||||
appType: IOS_APP
|
||||
appFile: ${{ matrix.test.ipaFile }}
|
||||
testFile: ${{ matrix.test.xcTestFile }}
|
||||
testPackageType: XCTEST_TEST_PACKAGE
|
||||
testType: XCTEST
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_ROLE_TO_ASSUME: ${{ vars.OIDC_AWS_ROLE_TO_ASSUME }}
|
||||
AWS_DEVICE_FARM_PROJECT_ARN: ${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }}
|
||||
AWS_DEVICE_FARM_DEVICE_POOL_ARN: ${{ vars.AWS_DEVICE_FARM_IPHONE_DEVICE_POOL_ARN }}
|
||||
run: |
|
||||
export name="${{ matrix.test.name }}"
|
||||
export appType=IOS_APP
|
||||
export appFile="${{ matrix.test.ipaFile }}"
|
||||
export testFile="${{ matrix.test.xcTestFile }}"
|
||||
export testPackageType=XCTEST_TEST_PACKAGE
|
||||
export testType=XCTEST
|
||||
export AWS_ACCESS_KEY_ID="${{ secrets.AWS_ACCESS_KEY_ID }}"
|
||||
export AWS_SECRET_ACCESS_KEY="${{ secrets.AWS_SECRET_ACCESS_KEY }}"
|
||||
export AWS_ROLE_TO_ASSUME="${{ vars.OIDC_AWS_ROLE_TO_ASSUME }}"
|
||||
export AWS_DEVICE_FARM_PROJECT_ARN="${{ vars.AWS_DEVICE_FARM_PROJECT_ARN }}"
|
||||
export AWS_DEVICE_FARM_DEVICE_POOL_ARN="${{ vars.AWS_DEVICE_FARM_IPHONE_DEVICE_POOL_ARN }}"
|
||||
export wait_for_completion=true
|
||||
|
||||
./scripts/aws-device-farm/aws-device-farm-run.sh
|
||||
|
||||
- uses: LouisBrunner/checks-action@v2.0.0
|
||||
if: always()
|
||||
|
||||
Reference in New Issue
Block a user