[build] Skip CI jobs if there are no relevant changed files

This commit is contained in:
Jason Wray
2019-08-08 15:35:56 -07:00
committed by GitHub
parent 56e7bbc0b7
commit a3a661e846
3 changed files with 196 additions and 23 deletions

View File

@@ -4,6 +4,14 @@ workflows:
version: 2
default:
jobs:
#
# Naming convention: {platform}-{additional description}-{build type}
# - {platform} is the client platform/framework, which may differ from
# the build platform. Specify both if applicable, e.g., "qt5-macos".
# - {additional description} optionally describes the compiler or other
# unique aspect of the build environment.
# - {build type} is typically "debug" or "release".
#
- nitpick
- clang-tidy:
filters:
@@ -467,6 +475,28 @@ commands:
scripts/notify-slack.sh
fi
#
# Add this step to all regular jobs to enable skipping of certain non-code-related changes.
#
# Do not include this step in nightly or release deployment jobs.
#
# To make a job potentially skippable on changes unrelated to its platform, it must:
# - Target one of the skippable platforms: Android, iOS, or macOS.
# - Have a job name that begins with a supported platform name.
# - Not be related to core functionality or rendering tests. Job names that
# contain "render-tests" cannot be skipped by platform changes.
#
# See the script in the following step for how to implement support for other platforms.
#
check-if-this-job-can-be-skipped:
steps:
- run:
name: Check if this job can be skipped
command: |
if [[ $CIRCLE_BRANCH != master ]] && [[ $CIRCLE_BRANCH != release-* ]]; then
scripts/check-ci-job-skippability.js
fi
jobs:
nitpick:
docker:
@@ -557,6 +587,7 @@ jobs:
MBGL_ANDROID_STL: << parameters.stl >>
steps:
- install-dependencies: { gradle: true }
- check-if-this-job-can-be-skipped
- run:
name: Initialize vendor submodules
command: git submodule update --init platform/android/vendor
@@ -633,6 +664,7 @@ jobs:
IS_LOCAL_DEVELOPMENT: false
steps:
- install-dependencies: { gradle: true }
- check-if-this-job-can-be-skipped
- run:
name: Initialize vendor submodules
command: git submodule update --init platform/android/vendor
@@ -723,6 +755,9 @@ jobs:
ANDROID_NDK: /android/sdk/ndk-bundle
steps:
- checkout
- npm-install
- prepare-environment
- check-if-this-job-can-be-skipped
- run:
name: Checkout submodules
command: |
@@ -766,6 +801,7 @@ jobs:
WITH_EGL: 1
steps:
- install-dependencies
- check-if-this-job-can-be-skipped
- build-node
- save-dependencies
- publish-node-package
@@ -782,6 +818,7 @@ jobs:
- install-macos-dependencies
- install-node-macos-dependencies
- install-dependencies
- check-if-this-job-can-be-skipped
- build-node
- save-dependencies
- run-node-macos-tests
@@ -803,6 +840,7 @@ jobs:
WITH_CXX11ABI: 1
steps:
- install-dependencies
- check-if-this-job-can-be-skipped
- build-linux
- save-dependencies
@@ -824,6 +862,7 @@ jobs:
UBSAN_OPTIONS: print_stacktrace=1:color=always:print_summary=1
steps:
- install-dependencies
- check-if-this-job-can-be-skipped
- setup-llvm-symbolizer
- build-test
- save-dependencies
@@ -846,6 +885,7 @@ jobs:
TSAN_OPTIONS: color=always:print_summary=1
steps:
- install-dependencies
- check-if-this-job-can-be-skipped
- setup-llvm-symbolizer
- build-test
- save-dependencies
@@ -866,6 +906,7 @@ jobs:
DISPLAY: :0
steps:
- install-dependencies
- check-if-this-job-can-be-skipped
- build-linux
- build-benchmark
- build-test
@@ -889,6 +930,7 @@ jobs:
WITH_COVERAGE: 1
steps:
- install-dependencies
- check-if-this-job-can-be-skipped
- build-linux
- build-benchmark
- build-test
@@ -919,6 +961,7 @@ jobs:
WITH_COVERAGE: 1
steps:
- install-dependencies
- check-if-this-job-can-be-skipped
- run:
name: Install doxygen
command: apt update && apt install -y doxygen
@@ -943,6 +986,7 @@ jobs:
WITH_EGL: 1
steps:
- install-dependencies
- check-if-this-job-can-be-skipped
- configure-cmake
- build-mbgl-render-test
- run-linux-render-tests
@@ -960,14 +1004,15 @@ jobs:
steps:
- install-macos-dependencies
- install-dependencies
- build-ios-test
- check-public-symbols
- run:
name: Check symbol namespacing for mapbox-events-ios
command: make ios-check-events-symbols
- run:
name: Lint podspecs and plist files
command: make ios-lint
- check-if-this-job-can-be-skipped
- build-ios-test
- run:
name: Check symbol namespacing for mapbox-events-ios
command: make ios-check-events-symbols
- run:
name: Nitpick Darwin code generation
command: scripts/nitpick/generated-code.js darwin
@@ -986,14 +1031,15 @@ jobs:
steps:
- install-macos-dependencies
- install-dependencies
- build-ios-test
- check-public-symbols
- run:
name: Check symbol namespacing for mapbox-events-ios
command: make ios-check-events-symbols
- run:
name: Lint podspecs and plist files
command: make ios-lint
- check-if-this-job-can-be-skipped
- build-ios-test
- run:
name: Check symbol namespacing for mapbox-events-ios
command: make ios-check-events-symbols
- run:
name: Nitpick Darwin code generation
command: scripts/nitpick/generated-code.js darwin
@@ -1001,21 +1047,6 @@ jobs:
- collect-xcode-build-logs
- upload-xcode-build-logs
# ------------------------------------------------------------------------------
metrics-nightly:
docker:
- image: mbgl/linux-gcc-5:54f59e3ac5
working_directory: /src
environment:
LIBSYSCONFCPUS: 2
JOBS: 2
steps:
- install-dependencies
- run:
name: Collect GitHub statistics
command: |
scripts/publish_github_stats.js
# ------------------------------------------------------------------------------
ios-sanitize-nightly:
macos:
@@ -1112,6 +1143,7 @@ jobs:
steps:
- install-macos-dependencies
- install-dependencies
- check-if-this-job-can-be-skipped
- install-ios-packaging-dependencies
- run:
name: Build dynamic framework for device and simulator
@@ -1210,6 +1242,7 @@ jobs:
steps:
- install-macos-dependencies
- install-dependencies
- check-if-this-job-can-be-skipped
- build-macos-test
- check-public-symbols
- run:
@@ -1237,12 +1270,28 @@ jobs:
steps:
- install-macos-dependencies
- install-dependencies
- check-if-this-job-can-be-skipped
- configure-cmake
- build-mbgl-render-test
- save-dependencies
- run-macos-render-tests
- upload-render-tests
# ------------------------------------------------------------------------------
metrics-nightly:
docker:
- image: mbgl/linux-gcc-5:54f59e3ac5
working_directory: /src
environment:
LIBSYSCONFCPUS: 2
JOBS: 2
steps:
- install-dependencies
- run:
name: Collect GitHub statistics
command: |
scripts/publish_github_stats.js
# ------------------------------------------------------------------------------
qt5-linux-gcc5-release:
docker:
@@ -1256,6 +1305,7 @@ jobs:
WITH_QT_I18N: 1
steps:
- install-dependencies
- check-if-this-job-can-be-skipped
- build-qt-app
- build-qt-test
- run:
@@ -1282,6 +1332,7 @@ jobs:
- install-macos-dependencies
- install-qt-macos-dependencies
- install-dependencies
- check-if-this-job-can-be-skipped
- build-qt-app
- build-qt-test
- run:

View File

@@ -14,6 +14,7 @@
"license": "BSD-2-Clause",
"dependencies": {
"@mapbox/cmake-node-module": "^1.2.0",
"minimatch": "^3.0.4",
"node-pre-gyp": "^0.10.2",
"npm-run-all": "^4.0.2"
},

View File

@@ -0,0 +1,121 @@
#!/usr/bin/env node
/*
This script takes two parameters or three environment variables:
{CI job name|$CIRCLE_JOB} {SHA1...SHA1|$CIRCLE_MERGE_BASE...$CIRCLE_SHA1}
The flow of this script:
- List of files changed in PR: `git diff --name-only start...end`
- Extract list into array.
- Remove items from array if they match:
- Global ignore list.
- Other platform ignore lists.
- If array still contains items, do not authorize skipping CI job.
- Else if array is empty, skip CI job.
To add support for a new platform:
- Add a new PLATFORM_FILES array with a list of unique paths for the
files belonging to the platform.
- Add a case to `skippableFilePatternsForCIJob()` with:
- The prefix for the platform's CI job names.
- A concatenated array of other platform paths that are safe to ignore
for the new platform. These should be files that can change without
having any effect on the output of the new platform.
- Update other cases in `skippableFilePatternsForCIJob()` to include the
paths for the new platform.
*/
const execSync = require('child_process').execSync;
const _ = require('lodash');
const minimatch = require("minimatch");
console.step = _.partial(console.log, '\n\033[1m\033[36m*', _, '\033[0m');
// BUILD PARAMETERS
const CI_JOB_NAME = process.argv[2] || process.env.CIRCLE_JOB;
if (!CI_JOB_NAME) { console.error('You must specify the name of a CI job as a parameter or environment variable.'); process.exit(1); }
const COMMIT_RANGE = process.argv[3] || `${process.env.CIRCLE_MERGE_BASE}...${process.env.CIRCLE_SHA1}`;
if (!COMMIT_RANGE || COMMIT_RANGE.includes('undefined')) { console.error('You must specify a range of commits to check for changed files in as a parameter or environment variable.'); process.exit(1); }
// FILE SKIPPING PATTERNS
const ALWAYS_SKIPPABLE_FILES = [
'**/CHANGELOG.md',
'**/README.md',
'**/CONTRIBUTING.md',
'**/DEVELOPING.md',
'**/INSTALL.md',
'**/*.podspec'
]
const ANDROID_FILES = [
'platform/android/**'
]
const DARWIN_FILES = [
'platform/darwin/**'
]
const IOS_FILES = [
'platform/ios/**'
]
const MACOS_FILES = [
'platform/macos/**'
]
const OTHER_PLATFORMS = [
'platform/glfw/**',
'platform/linux/**',
'platform/node/**',
'platform/qt/**',
'benchmark/**',
'test/**'
]
function skippableFilePatternsForCIJob(job) {
if (job.includes('render-tests')) {
// Render tests run on specific platforms are effectively unskippable core jobs, not platform jobs.
return ALWAYS_SKIPPABLE_FILES;
} else if (job.startsWith('ios')) {
return _.concat(ALWAYS_SKIPPABLE_FILES, ANDROID_FILES, MACOS_FILES, OTHER_PLATFORMS);
} else if (job.startsWith('android')) {
return _.concat(ALWAYS_SKIPPABLE_FILES, DARWIN_FILES, IOS_FILES, MACOS_FILES, OTHER_PLATFORMS);
} else if (job.startsWith('macos')) {
return _.concat(ALWAYS_SKIPPABLE_FILES, ANDROID_FILES, IOS_FILES, OTHER_PLATFORMS);
} else {
return ALWAYS_SKIPPABLE_FILES;
}
}
console.step(`Getting list of files changed between ${COMMIT_RANGE}`);
let changedFiles = execSync(`git diff --name-only ${COMMIT_RANGE}`).toString().split('\n');
changedFiles = _.compact(changedFiles);
console.log(`${changedFiles.length} total files changed.`);
console.step(`Checking changed files for paths relevant to ${CI_JOB_NAME} job`);
// Filter the changed files array to remove files that are irrelevant to the specified CI job.
_.remove(changedFiles, function(changedFile) {
for (const skippableFilePattern of skippableFilePatternsForCIJob(CI_JOB_NAME)) {
if (minimatch(changedFile, skippableFilePattern, { dot: true, nocase: true })) {
return true;
}
}
return false;
});
if (changedFiles.length) {
console.log(`Found ${changedFiles.length} unskippable changed file${changedFiles.length > 1 ? 's':''}:\n`, changedFiles);
} else {
console.log(`Found no relevant changed files, so it's safe to skip the remainder of this CI job.`)
if (process.env.CIRCLECI) {
console.step('Aborting CI job');
execSync(`circleci step halt`, {stdio: 'inherit'});
}
}