[build] Skip CI jobs if there are no relevant changed files
This commit is contained in:
97
circle.yml
97
circle.yml
@@ -4,6 +4,14 @@ workflows:
|
||||
version: 2
|
||||
default:
|
||||
jobs:
|
||||
#
|
||||
# Naming convention: {platform}-{additional description}-{build type}
|
||||
# - {platform} is the client platform/framework, which may differ from
|
||||
# the build platform. Specify both if applicable, e.g., "qt5-macos".
|
||||
# - {additional description} optionally describes the compiler or other
|
||||
# unique aspect of the build environment.
|
||||
# - {build type} is typically "debug" or "release".
|
||||
#
|
||||
- nitpick
|
||||
- clang-tidy:
|
||||
filters:
|
||||
@@ -467,6 +475,28 @@ commands:
|
||||
scripts/notify-slack.sh
|
||||
fi
|
||||
|
||||
#
|
||||
# Add this step to all regular jobs to enable skipping of certain non-code-related changes.
|
||||
#
|
||||
# Do not include this step in nightly or release deployment jobs.
|
||||
#
|
||||
# To make a job potentially skippable on changes unrelated to its platform, it must:
|
||||
# - Target one of the skippable platforms: Android, iOS, or macOS.
|
||||
# - Have a job name that begins with a supported platform name.
|
||||
# - Not be related to core functionality or rendering tests. Job names that
|
||||
# contain "render-tests" cannot be skipped by platform changes.
|
||||
#
|
||||
# See the script in the following step for how to implement support for other platforms.
|
||||
#
|
||||
check-if-this-job-can-be-skipped:
|
||||
steps:
|
||||
- run:
|
||||
name: Check if this job can be skipped
|
||||
command: |
|
||||
if [[ $CIRCLE_BRANCH != master ]] && [[ $CIRCLE_BRANCH != release-* ]]; then
|
||||
scripts/check-ci-job-skippability.js
|
||||
fi
|
||||
|
||||
jobs:
|
||||
nitpick:
|
||||
docker:
|
||||
@@ -557,6 +587,7 @@ jobs:
|
||||
MBGL_ANDROID_STL: << parameters.stl >>
|
||||
steps:
|
||||
- install-dependencies: { gradle: true }
|
||||
- check-if-this-job-can-be-skipped
|
||||
- run:
|
||||
name: Initialize vendor submodules
|
||||
command: git submodule update --init platform/android/vendor
|
||||
@@ -633,6 +664,7 @@ jobs:
|
||||
IS_LOCAL_DEVELOPMENT: false
|
||||
steps:
|
||||
- install-dependencies: { gradle: true }
|
||||
- check-if-this-job-can-be-skipped
|
||||
- run:
|
||||
name: Initialize vendor submodules
|
||||
command: git submodule update --init platform/android/vendor
|
||||
@@ -723,6 +755,9 @@ jobs:
|
||||
ANDROID_NDK: /android/sdk/ndk-bundle
|
||||
steps:
|
||||
- checkout
|
||||
- npm-install
|
||||
- prepare-environment
|
||||
- check-if-this-job-can-be-skipped
|
||||
- run:
|
||||
name: Checkout submodules
|
||||
command: |
|
||||
@@ -766,6 +801,7 @@ jobs:
|
||||
WITH_EGL: 1
|
||||
steps:
|
||||
- install-dependencies
|
||||
- check-if-this-job-can-be-skipped
|
||||
- build-node
|
||||
- save-dependencies
|
||||
- publish-node-package
|
||||
@@ -782,6 +818,7 @@ jobs:
|
||||
- install-macos-dependencies
|
||||
- install-node-macos-dependencies
|
||||
- install-dependencies
|
||||
- check-if-this-job-can-be-skipped
|
||||
- build-node
|
||||
- save-dependencies
|
||||
- run-node-macos-tests
|
||||
@@ -803,6 +840,7 @@ jobs:
|
||||
WITH_CXX11ABI: 1
|
||||
steps:
|
||||
- install-dependencies
|
||||
- check-if-this-job-can-be-skipped
|
||||
- build-linux
|
||||
- save-dependencies
|
||||
|
||||
@@ -824,6 +862,7 @@ jobs:
|
||||
UBSAN_OPTIONS: print_stacktrace=1:color=always:print_summary=1
|
||||
steps:
|
||||
- install-dependencies
|
||||
- check-if-this-job-can-be-skipped
|
||||
- setup-llvm-symbolizer
|
||||
- build-test
|
||||
- save-dependencies
|
||||
@@ -846,6 +885,7 @@ jobs:
|
||||
TSAN_OPTIONS: color=always:print_summary=1
|
||||
steps:
|
||||
- install-dependencies
|
||||
- check-if-this-job-can-be-skipped
|
||||
- setup-llvm-symbolizer
|
||||
- build-test
|
||||
- save-dependencies
|
||||
@@ -866,6 +906,7 @@ jobs:
|
||||
DISPLAY: :0
|
||||
steps:
|
||||
- install-dependencies
|
||||
- check-if-this-job-can-be-skipped
|
||||
- build-linux
|
||||
- build-benchmark
|
||||
- build-test
|
||||
@@ -889,6 +930,7 @@ jobs:
|
||||
WITH_COVERAGE: 1
|
||||
steps:
|
||||
- install-dependencies
|
||||
- check-if-this-job-can-be-skipped
|
||||
- build-linux
|
||||
- build-benchmark
|
||||
- build-test
|
||||
@@ -919,6 +961,7 @@ jobs:
|
||||
WITH_COVERAGE: 1
|
||||
steps:
|
||||
- install-dependencies
|
||||
- check-if-this-job-can-be-skipped
|
||||
- run:
|
||||
name: Install doxygen
|
||||
command: apt update && apt install -y doxygen
|
||||
@@ -943,6 +986,7 @@ jobs:
|
||||
WITH_EGL: 1
|
||||
steps:
|
||||
- install-dependencies
|
||||
- check-if-this-job-can-be-skipped
|
||||
- configure-cmake
|
||||
- build-mbgl-render-test
|
||||
- run-linux-render-tests
|
||||
@@ -960,14 +1004,15 @@ jobs:
|
||||
steps:
|
||||
- install-macos-dependencies
|
||||
- install-dependencies
|
||||
- build-ios-test
|
||||
- check-public-symbols
|
||||
- run:
|
||||
name: Check symbol namespacing for mapbox-events-ios
|
||||
command: make ios-check-events-symbols
|
||||
- run:
|
||||
name: Lint podspecs and plist files
|
||||
command: make ios-lint
|
||||
- check-if-this-job-can-be-skipped
|
||||
- build-ios-test
|
||||
- run:
|
||||
name: Check symbol namespacing for mapbox-events-ios
|
||||
command: make ios-check-events-symbols
|
||||
- run:
|
||||
name: Nitpick Darwin code generation
|
||||
command: scripts/nitpick/generated-code.js darwin
|
||||
@@ -986,14 +1031,15 @@ jobs:
|
||||
steps:
|
||||
- install-macos-dependencies
|
||||
- install-dependencies
|
||||
- build-ios-test
|
||||
- check-public-symbols
|
||||
- run:
|
||||
name: Check symbol namespacing for mapbox-events-ios
|
||||
command: make ios-check-events-symbols
|
||||
- run:
|
||||
name: Lint podspecs and plist files
|
||||
command: make ios-lint
|
||||
- check-if-this-job-can-be-skipped
|
||||
- build-ios-test
|
||||
- run:
|
||||
name: Check symbol namespacing for mapbox-events-ios
|
||||
command: make ios-check-events-symbols
|
||||
- run:
|
||||
name: Nitpick Darwin code generation
|
||||
command: scripts/nitpick/generated-code.js darwin
|
||||
@@ -1001,21 +1047,6 @@ jobs:
|
||||
- collect-xcode-build-logs
|
||||
- upload-xcode-build-logs
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
metrics-nightly:
|
||||
docker:
|
||||
- image: mbgl/linux-gcc-5:54f59e3ac5
|
||||
working_directory: /src
|
||||
environment:
|
||||
LIBSYSCONFCPUS: 2
|
||||
JOBS: 2
|
||||
steps:
|
||||
- install-dependencies
|
||||
- run:
|
||||
name: Collect GitHub statistics
|
||||
command: |
|
||||
scripts/publish_github_stats.js
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
ios-sanitize-nightly:
|
||||
macos:
|
||||
@@ -1112,6 +1143,7 @@ jobs:
|
||||
steps:
|
||||
- install-macos-dependencies
|
||||
- install-dependencies
|
||||
- check-if-this-job-can-be-skipped
|
||||
- install-ios-packaging-dependencies
|
||||
- run:
|
||||
name: Build dynamic framework for device and simulator
|
||||
@@ -1210,6 +1242,7 @@ jobs:
|
||||
steps:
|
||||
- install-macos-dependencies
|
||||
- install-dependencies
|
||||
- check-if-this-job-can-be-skipped
|
||||
- build-macos-test
|
||||
- check-public-symbols
|
||||
- run:
|
||||
@@ -1237,12 +1270,28 @@ jobs:
|
||||
steps:
|
||||
- install-macos-dependencies
|
||||
- install-dependencies
|
||||
- check-if-this-job-can-be-skipped
|
||||
- configure-cmake
|
||||
- build-mbgl-render-test
|
||||
- save-dependencies
|
||||
- run-macos-render-tests
|
||||
- upload-render-tests
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
metrics-nightly:
|
||||
docker:
|
||||
- image: mbgl/linux-gcc-5:54f59e3ac5
|
||||
working_directory: /src
|
||||
environment:
|
||||
LIBSYSCONFCPUS: 2
|
||||
JOBS: 2
|
||||
steps:
|
||||
- install-dependencies
|
||||
- run:
|
||||
name: Collect GitHub statistics
|
||||
command: |
|
||||
scripts/publish_github_stats.js
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
qt5-linux-gcc5-release:
|
||||
docker:
|
||||
@@ -1256,6 +1305,7 @@ jobs:
|
||||
WITH_QT_I18N: 1
|
||||
steps:
|
||||
- install-dependencies
|
||||
- check-if-this-job-can-be-skipped
|
||||
- build-qt-app
|
||||
- build-qt-test
|
||||
- run:
|
||||
@@ -1282,6 +1332,7 @@ jobs:
|
||||
- install-macos-dependencies
|
||||
- install-qt-macos-dependencies
|
||||
- install-dependencies
|
||||
- check-if-this-job-can-be-skipped
|
||||
- build-qt-app
|
||||
- build-qt-test
|
||||
- run:
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"@mapbox/cmake-node-module": "^1.2.0",
|
||||
"minimatch": "^3.0.4",
|
||||
"node-pre-gyp": "^0.10.2",
|
||||
"npm-run-all": "^4.0.2"
|
||||
},
|
||||
|
||||
121
scripts/check-ci-job-skippability.js
Executable file
121
scripts/check-ci-job-skippability.js
Executable file
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/*
|
||||
This script takes two parameters or three environment variables:
|
||||
|
||||
{CI job name|$CIRCLE_JOB} {SHA1...SHA1|$CIRCLE_MERGE_BASE...$CIRCLE_SHA1}
|
||||
|
||||
The flow of this script:
|
||||
|
||||
- List of files changed in PR: `git diff --name-only start...end`
|
||||
- Extract list into array.
|
||||
- Remove items from array if they match:
|
||||
- Global ignore list.
|
||||
- Other platform ignore lists.
|
||||
- If array still contains items, do not authorize skipping CI job.
|
||||
- Else if array is empty, skip CI job.
|
||||
|
||||
To add support for a new platform:
|
||||
|
||||
- Add a new PLATFORM_FILES array with a list of unique paths for the
|
||||
files belonging to the platform.
|
||||
- Add a case to `skippableFilePatternsForCIJob()` with:
|
||||
- The prefix for the platform's CI job names.
|
||||
- A concatenated array of other platform paths that are safe to ignore
|
||||
for the new platform. These should be files that can change without
|
||||
having any effect on the output of the new platform.
|
||||
- Update other cases in `skippableFilePatternsForCIJob()` to include the
|
||||
paths for the new platform.
|
||||
*/
|
||||
|
||||
const execSync = require('child_process').execSync;
|
||||
const _ = require('lodash');
|
||||
const minimatch = require("minimatch");
|
||||
|
||||
console.step = _.partial(console.log, '\n\033[1m\033[36m*', _, '\033[0m');
|
||||
|
||||
// BUILD PARAMETERS
|
||||
const CI_JOB_NAME = process.argv[2] || process.env.CIRCLE_JOB;
|
||||
if (!CI_JOB_NAME) { console.error('You must specify the name of a CI job as a parameter or environment variable.'); process.exit(1); }
|
||||
|
||||
const COMMIT_RANGE = process.argv[3] || `${process.env.CIRCLE_MERGE_BASE}...${process.env.CIRCLE_SHA1}`;
|
||||
if (!COMMIT_RANGE || COMMIT_RANGE.includes('undefined')) { console.error('You must specify a range of commits to check for changed files in as a parameter or environment variable.'); process.exit(1); }
|
||||
|
||||
// FILE SKIPPING PATTERNS
|
||||
const ALWAYS_SKIPPABLE_FILES = [
|
||||
'**/CHANGELOG.md',
|
||||
'**/README.md',
|
||||
'**/CONTRIBUTING.md',
|
||||
'**/DEVELOPING.md',
|
||||
'**/INSTALL.md',
|
||||
'**/*.podspec'
|
||||
]
|
||||
|
||||
const ANDROID_FILES = [
|
||||
'platform/android/**'
|
||||
]
|
||||
|
||||
const DARWIN_FILES = [
|
||||
'platform/darwin/**'
|
||||
]
|
||||
|
||||
const IOS_FILES = [
|
||||
'platform/ios/**'
|
||||
]
|
||||
|
||||
const MACOS_FILES = [
|
||||
'platform/macos/**'
|
||||
]
|
||||
|
||||
const OTHER_PLATFORMS = [
|
||||
'platform/glfw/**',
|
||||
'platform/linux/**',
|
||||
'platform/node/**',
|
||||
'platform/qt/**',
|
||||
'benchmark/**',
|
||||
'test/**'
|
||||
]
|
||||
|
||||
function skippableFilePatternsForCIJob(job) {
|
||||
if (job.includes('render-tests')) {
|
||||
// Render tests run on specific platforms are effectively unskippable core jobs, not platform jobs.
|
||||
return ALWAYS_SKIPPABLE_FILES;
|
||||
} else if (job.startsWith('ios')) {
|
||||
return _.concat(ALWAYS_SKIPPABLE_FILES, ANDROID_FILES, MACOS_FILES, OTHER_PLATFORMS);
|
||||
} else if (job.startsWith('android')) {
|
||||
return _.concat(ALWAYS_SKIPPABLE_FILES, DARWIN_FILES, IOS_FILES, MACOS_FILES, OTHER_PLATFORMS);
|
||||
} else if (job.startsWith('macos')) {
|
||||
return _.concat(ALWAYS_SKIPPABLE_FILES, ANDROID_FILES, IOS_FILES, OTHER_PLATFORMS);
|
||||
} else {
|
||||
return ALWAYS_SKIPPABLE_FILES;
|
||||
}
|
||||
}
|
||||
|
||||
console.step(`Getting list of files changed between ${COMMIT_RANGE}`);
|
||||
|
||||
let changedFiles = execSync(`git diff --name-only ${COMMIT_RANGE}`).toString().split('\n');
|
||||
changedFiles = _.compact(changedFiles);
|
||||
|
||||
console.log(`${changedFiles.length} total files changed.`);
|
||||
|
||||
console.step(`Checking changed files for paths relevant to ${CI_JOB_NAME} job`);
|
||||
|
||||
// Filter the changed files array to remove files that are irrelevant to the specified CI job.
|
||||
_.remove(changedFiles, function(changedFile) {
|
||||
for (const skippableFilePattern of skippableFilePatternsForCIJob(CI_JOB_NAME)) {
|
||||
if (minimatch(changedFile, skippableFilePattern, { dot: true, nocase: true })) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
});
|
||||
|
||||
if (changedFiles.length) {
|
||||
console.log(`Found ${changedFiles.length} unskippable changed file${changedFiles.length > 1 ? 's':''}:\n`, changedFiles);
|
||||
} else {
|
||||
console.log(`Found no relevant changed files, so it's safe to skip the remainder of this CI job.`)
|
||||
if (process.env.CIRCLECI) {
|
||||
console.step('Aborting CI job');
|
||||
execSync(`circleci step halt`, {stdio: 'inherit'});
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user