Cleanup more scripts (#835)
This commit is contained in:
Bart Louwers
@@ -1,124 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
console.log("DISABLED — https://github.com/mapbox/mapbox-gl-native-ios/issues/19")
|
||||
process.exit(0)
|
||||
|
||||
/*
|
||||
This script takes two parameters or three environment variables:
|
||||
|
||||
{CI job name|$CIRCLE_JOB} {SHA1...SHA1|$CIRCLE_MERGE_BASE...$CIRCLE_SHA1}
|
||||
|
||||
The flow of this script:
|
||||
|
||||
- List of files changed in PR: `git diff --name-only start...end`
|
||||
- Extract list into array.
|
||||
- Remove items from array if they match:
|
||||
- Global ignore list.
|
||||
- Other platform ignore lists.
|
||||
- If array still contains items, do not authorize skipping CI job.
|
||||
- Else if array is empty, skip CI job.
|
||||
|
||||
To add support for a new platform:
|
||||
|
||||
- Add a new PLATFORM_FILES array with a list of unique paths for the
|
||||
files belonging to the platform.
|
||||
- Add a case to `skippableFilePatternsForCIJob()` with:
|
||||
- The prefix for the platform's CI job names.
|
||||
- A concatenated array of other platform paths that are safe to ignore
|
||||
for the new platform. These should be files that can change without
|
||||
having any effect on the output of the new platform.
|
||||
- Update other cases in `skippableFilePatternsForCIJob()` to include the
|
||||
paths for the new platform.
|
||||
*/
|
||||
|
||||
const execSync = require('child_process').execSync;
|
||||
const _ = require('lodash');
|
||||
const minimatch = require("minimatch");
|
||||
|
||||
console.step = _.partial(console.log, '\n\033[1m\033[36m*', _, '\033[0m');
|
||||
|
||||
// BUILD PARAMETERS
|
||||
const CI_JOB_NAME = process.argv[2] || process.env.CIRCLE_JOB;
|
||||
if (!CI_JOB_NAME) { console.error('You must specify the name of a CI job as a parameter or environment variable.'); process.exit(1); }
|
||||
|
||||
const COMMIT_RANGE = process.argv[3] || `${process.env.CIRCLE_MERGE_BASE}...${process.env.CIRCLE_SHA1}`;
|
||||
if (!COMMIT_RANGE || COMMIT_RANGE.includes('undefined')) { console.error('You must specify a range of commits to check for changed files in as a parameter or environment variable.'); process.exit(1); }
|
||||
|
||||
// FILE SKIPPING PATTERNS
|
||||
const ALWAYS_SKIPPABLE_FILES = [
|
||||
'**/CHANGELOG.md',
|
||||
'**/README.md',
|
||||
'**/CONTRIBUTING.md',
|
||||
'**/DEVELOPING.md',
|
||||
'**/INSTALL.md',
|
||||
'**/*.podspec'
|
||||
]
|
||||
|
||||
const ANDROID_FILES = [
|
||||
'platform/android/**'
|
||||
]
|
||||
|
||||
const DARWIN_FILES = [
|
||||
'platform/darwin/**'
|
||||
]
|
||||
|
||||
const IOS_FILES = [
|
||||
'platform/ios/**'
|
||||
]
|
||||
|
||||
const MACOS_FILES = [
|
||||
'platform/macos/**'
|
||||
]
|
||||
|
||||
const OTHER_PLATFORMS = [
|
||||
'platform/glfw/**',
|
||||
'platform/linux/**',
|
||||
'platform/node/**',
|
||||
'platform/qt/**',
|
||||
'benchmark/**',
|
||||
'test/**'
|
||||
]
|
||||
|
||||
function skippableFilePatternsForCIJob(job) {
|
||||
if (job.includes('render-tests')) {
|
||||
// Render tests run on specific platforms are effectively unskippable core jobs, not platform jobs.
|
||||
return ALWAYS_SKIPPABLE_FILES;
|
||||
} else if (job.startsWith('ios')) {
|
||||
return _.concat(ALWAYS_SKIPPABLE_FILES, ANDROID_FILES, MACOS_FILES, OTHER_PLATFORMS);
|
||||
} else if (job.startsWith('android')) {
|
||||
return _.concat(ALWAYS_SKIPPABLE_FILES, DARWIN_FILES, IOS_FILES, MACOS_FILES, OTHER_PLATFORMS);
|
||||
} else if (job.startsWith('macos')) {
|
||||
return _.concat(ALWAYS_SKIPPABLE_FILES, ANDROID_FILES, IOS_FILES, OTHER_PLATFORMS);
|
||||
} else {
|
||||
return ALWAYS_SKIPPABLE_FILES;
|
||||
}
|
||||
}
|
||||
|
||||
console.step(`Getting list of files changed between ${COMMIT_RANGE}`);
|
||||
|
||||
let changedFiles = execSync(`git diff --name-only ${COMMIT_RANGE}`).toString().split('\n');
|
||||
changedFiles = _.compact(changedFiles);
|
||||
|
||||
console.log(`${changedFiles.length} file${changedFiles.length == 1 ? '':'s'} changed.`);
|
||||
|
||||
console.step(`Checking changed files for paths relevant to ${CI_JOB_NAME} job`);
|
||||
|
||||
// Filter the changed files array to remove files that are irrelevant to the specified CI job.
|
||||
_.remove(changedFiles, function(changedFile) {
|
||||
for (const skippableFilePattern of skippableFilePatternsForCIJob(CI_JOB_NAME)) {
|
||||
if (minimatch(changedFile, skippableFilePattern, { dot: true, nocase: true })) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
});
|
||||
|
||||
if (changedFiles.length) {
|
||||
console.log(`Found ${changedFiles.length} unskippable changed file${changedFiles.length == 1 ? '':'s'}:\n`, changedFiles);
|
||||
} else {
|
||||
console.log(`Found no relevant changed files, so it's safe to skip the remainder of this CI job.`)
|
||||
if (process.env.CIRCLECI) {
|
||||
console.step('Aborting CI job');
|
||||
execSync(`circleci step halt`, {stdio: 'inherit'});
|
||||
}
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
if [ ! $(command -v qmake 2> /dev/null) ]; then
|
||||
echo "No qmake found - missing Qt development files"
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,7 +0,0 @@
|
||||
param([string]$file, [string]$hash);
|
||||
|
||||
$hashFromFile = Get-FileHash -Path $file -Algorithm SHA256
|
||||
|
||||
if ($hashFromFile.Hash -ne $hash) {
|
||||
Throw 'Hash mismatch!'
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const jwt = require('jsonwebtoken');
|
||||
const github = require('@octokit/rest').plugin(require('@octokit/plugin-retry'))({
|
||||
retry: { doNotRetry: [ /* Empty — retry on any error code. */ ] }
|
||||
})
|
||||
const prettyBytes = require('pretty-bytes');
|
||||
const fs = require('fs');
|
||||
|
||||
const SIZE_CHECK_APP_ID = 14028;
|
||||
const SIZE_CHECK_APP_INSTALLATION_ID = 229425;
|
||||
|
||||
const file = process.argv[2];
|
||||
const label = process.argv[3];
|
||||
const {size} = fs.statSync(file);
|
||||
const name = `Size - ${label}`;
|
||||
|
||||
process.on('unhandledRejection', error => {
|
||||
console.log(error);
|
||||
process.exit(1)
|
||||
});
|
||||
|
||||
const pk = process.env['SIZE_CHECK_APP_PRIVATE_KEY'];
|
||||
if (!pk) {
|
||||
console.log('Fork PR; not computing size.');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const key = Buffer.from(pk, 'base64').toString('binary');
|
||||
const payload = {
|
||||
exp: Math.floor(Date.now() / 1000) + 60,
|
||||
iat: Math.floor(Date.now() / 1000),
|
||||
iss: SIZE_CHECK_APP_ID
|
||||
};
|
||||
|
||||
const token = jwt.sign(payload, key, {algorithm: 'RS256'});
|
||||
github.authenticate({type: 'app', token});
|
||||
|
||||
function getPriorSize() {
|
||||
const mergeBase = process.env['CIRCLE_MERGE_BASE'];
|
||||
if (!mergeBase) {
|
||||
console.log('No merge base available.');
|
||||
return Promise.resolve(null);
|
||||
}
|
||||
|
||||
return github.checks.listForRef({
|
||||
owner: 'mapbox',
|
||||
repo: 'mapbox-gl-native',
|
||||
ref: mergeBase
|
||||
}).then(({data}) => {
|
||||
const run = data.check_runs.find(run => run.name === name);
|
||||
if (!run) {
|
||||
console.log('No matching check found.');
|
||||
return Promise.resolve(null);
|
||||
}
|
||||
return +run.output.summary.match(/`.*` is (\d+) bytes/)[1];
|
||||
});
|
||||
}
|
||||
|
||||
github.apps.createInstallationToken({installation_id: SIZE_CHECK_APP_INSTALLATION_ID})
|
||||
.then(({data}) => {
|
||||
github.authenticate({type: 'token', token: data.token});
|
||||
getPriorSize().then(prior => {
|
||||
const title = (() => {
|
||||
if (prior) {
|
||||
const change = size - prior;
|
||||
const percent = (change / prior) * 100;
|
||||
return `${change >= 0 ? '+' : ''}${prettyBytes(change)} ${percent.toFixed(3)}% (${prettyBytes(size)})`;
|
||||
} else {
|
||||
return prettyBytes(size);
|
||||
}
|
||||
})();
|
||||
|
||||
console.log(`${label}: ${title} (${size} bytes)`);
|
||||
|
||||
return github.checks.create({
|
||||
owner: 'mapbox',
|
||||
repo: 'mapbox-gl-native',
|
||||
name: `Size - ${label}`,
|
||||
head_branch: process.env['CIRCLE_BRANCH'],
|
||||
head_sha: process.env['CIRCLE_SHA1'],
|
||||
status: 'completed',
|
||||
conclusion: 'success',
|
||||
completed_at: new Date().toISOString(),
|
||||
output: {
|
||||
title,
|
||||
summary: `\`${file}\` is ${size} bytes (${prettyBytes(size)})`
|
||||
}
|
||||
});
|
||||
})
|
||||
});
|
||||
@@ -1,71 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Implements https://circleci.com/docs/api/v2/#trigger-a-new-pipeline
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import requests
|
||||
import sys
|
||||
|
||||
|
||||
def TriggerPiperline(slug, token, branch, params):
|
||||
url = "https://circleci.com/api/v2/project/github/%s/pipeline" % (slug)
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
|
||||
data = {
|
||||
"parameters": params
|
||||
}
|
||||
|
||||
if branch:
|
||||
data["branch"] = branch
|
||||
|
||||
r = requests.post(url, auth=(token, ""), headers=headers, json=data)
|
||||
|
||||
if r.status_code != 201 and r.status_code != 200:
|
||||
print("Error triggering the CircleCI: %s." % r.json()["message"])
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def Main():
|
||||
token = os.getenv("CIRCLE_API_TOKEN")
|
||||
hash = os.getenv("CIRCLE_SHA1", default="1234567890")
|
||||
username = os.getenv("CIRCLE_PROJECT_USERNAME", default="username")
|
||||
project = os.getenv("CIRCLE_PROJECT_REPONAME", default="repository")
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Creates CircleCI jobs and waits for the result.")
|
||||
|
||||
parser.add_argument("--token", default=token,
|
||||
help="CircleCI token, otherwise environment CIRCLE_API_TOKEN.")
|
||||
parser.add_argument("--origin-slug", default="mapbox/mapbox-gl-native",
|
||||
help="Origin repository, otherwise CIRCLE_PROJECT_USERNAME/CIRCLE_PROJECT_REPONAME.")
|
||||
parser.add_argument("--target-slug", default="".join((username, '/', project)),
|
||||
help="Repository to trigger the pipeline, example: mapbox/mapbox-gl-native-internal.")
|
||||
parser.add_argument("--hash", default=hash,
|
||||
help="Commit git hash that triggered the pipeline, otherwise environment CIRCLE_SHA1.")
|
||||
parser.add_argument("--branch",
|
||||
help="Build a specific branch, otherwise it will build the default branch.")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.token:
|
||||
print("CircleCI token not set. Use --token or set CIRCLE_API_TOKEN.")
|
||||
sys.exit(1)
|
||||
|
||||
params = {
|
||||
"mapbox_upstream": True,
|
||||
"mapbox_slug": args.origin_slug,
|
||||
"mapbox_hash": args.hash
|
||||
}
|
||||
|
||||
TriggerPiperline(args.target_slug, args.token, args.branch, params)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
Main()
|
||||
@@ -1,133 +0,0 @@
|
||||
FROM ubuntu:eoan-20200313
|
||||
|
||||
ENV DEBIAN_FRONTEND="noninteractive"
|
||||
ENV ANDROID_HOME="/opt/android"
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
# Use faster mirrors from Finland
|
||||
COPY etc/apt/sources.list /etc/apt
|
||||
|
||||
RUN set -eu \
|
||||
&& apt-get update \
|
||||
&& apt-get dist-upgrade -y \
|
||||
&& apt-get autoremove --purge -y && apt-get clean
|
||||
|
||||
# CI requirements
|
||||
RUN set -eu && apt-get install -y \
|
||||
ca-certificates \
|
||||
git \
|
||||
gzip \
|
||||
ssh \
|
||||
tar \
|
||||
colorized-logs \
|
||||
python-scipy \
|
||||
&& apt-get autoremove --purge -y && apt-get clean
|
||||
|
||||
# Base dependencies
|
||||
RUN set -eu && apt-get install -y \
|
||||
ccache \
|
||||
clang-8 \
|
||||
clang-format-8 \
|
||||
clang-tidy-8 \
|
||||
cmake \
|
||||
fonts-noto \
|
||||
g++-8 \
|
||||
libc++-8-dev \
|
||||
libc++abi-8-dev \
|
||||
mesa-common-dev \
|
||||
ninja-build \
|
||||
nodejs \
|
||||
npm \
|
||||
pkg-config \
|
||||
python3-pip \
|
||||
python3-requests \
|
||||
python3-github \
|
||||
software-properties-common \
|
||||
valgrind \
|
||||
xvfb \
|
||||
&& apt-get autoremove --purge -y && apt-get clean
|
||||
|
||||
RUN pip3 install cmake-format==0.5.5
|
||||
|
||||
# Linux dependencies
|
||||
RUN set -eu && apt-get install -y \
|
||||
libcurl4-openssl-dev \
|
||||
libgl1-mesa-dev \
|
||||
libgles2-mesa-dev \
|
||||
libglfw3-dev \
|
||||
libicu-dev \
|
||||
libjpeg-turbo8-dev \
|
||||
libpng-dev \
|
||||
libuv1-dev \
|
||||
zlib1g-dev \
|
||||
&& apt-get autoremove --purge -y && apt-get clean
|
||||
|
||||
# Qt dependencies
|
||||
RUN set -eu && apt-get install -y \
|
||||
qdoc-qt5 \
|
||||
qt5-default \
|
||||
&& apt-get autoremove --purge -y && apt-get clean
|
||||
|
||||
# Android dependencies
|
||||
RUN set -eu && apt-get install -y \
|
||||
coreutils \
|
||||
curl \
|
||||
openjdk-8-jdk-headless \
|
||||
unzip \
|
||||
&& apt-get autoremove --purge -y && apt-get clean
|
||||
|
||||
# Install old compilers
|
||||
RUN set -eu \
|
||||
&& apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 1E9377A2BA9EF27F \
|
||||
&& add-apt-repository "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu xenial main" \
|
||||
&& add-apt-repository "deb http://us.archive.ubuntu.com/ubuntu/ xenial main universe" \
|
||||
&& apt-get install -y g++-5 \
|
||||
&& apt-get autoremove --purge -y && apt-get clean
|
||||
|
||||
# Install Android NDK
|
||||
RUN set -eu \
|
||||
&& mkdir -p ${ANDROID_HOME} && cd ${ANDROID_HOME} \
|
||||
&& curl -L --retry 3 https://dl.google.com/android/repository/android-ndk-r19-linux-x86_64.zip -o ndk.zip \
|
||||
&& (echo "f02ad84cb5b6e1ff3eea9e6168037c823408c8ac ndk.zip" | sha1sum -c) \
|
||||
&& unzip -q ndk.zip && rm ndk.zip && mv android-ndk-r* ndk-bundle
|
||||
|
||||
|
||||
RUN set -eu \
|
||||
&& cd ${ANDROID_HOME} \
|
||||
&& curl -L --retry 3 https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip -o tools.zip \
|
||||
&& (echo "92ffee5a1d98d856634e8b71132e8a95d96c83a63fde1099be3d86df3106def9 tools.zip" | sha256sum -c) \
|
||||
&& unzip -q tools.zip && rm tools.zip
|
||||
|
||||
RUN set -eu \
|
||||
&& yes | ${ANDROID_HOME}/tools/bin/sdkmanager \
|
||||
"platform-tools" \
|
||||
"platforms;android-26" \
|
||||
"build-tools;26.0.3" \
|
||||
"platforms;android-27" \
|
||||
"build-tools;27.0.3" \
|
||||
"platforms;android-28" \
|
||||
"build-tools;28.0.3" \
|
||||
"extras;android;m2repository" \
|
||||
"patcher;v4" \
|
||||
"extras;google;m2repository" \
|
||||
"extras;m2repository;com;android;support;constraint;constraint-layout;1.0.2" \
|
||||
"cmake;3.10.2.4988404"
|
||||
|
||||
# Install gcloud for Firebase testing
|
||||
RUN set -eu \
|
||||
&& echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list \
|
||||
&& curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - \
|
||||
&& apt-get update -y \
|
||||
&& apt-get install google-cloud-sdk -y \
|
||||
&& apt-get autoremove --purge -y && apt-get clean
|
||||
|
||||
# FIXME: This package should not be needed
|
||||
# bug looks like run-clang-tidy-8 is broken.
|
||||
# Remove when Ubuntu Eoan fixes the dependency.
|
||||
RUN set -eu && apt-get install -y \
|
||||
python-yaml \
|
||||
&& apt-get autoremove --purge -y && apt-get clean
|
||||
|
||||
# Configure ccache
|
||||
RUN set -eu && /usr/sbin/update-ccache-symlinks
|
||||
@@ -1,10 +0,0 @@
|
||||
deb http://fi.archive.ubuntu.com/ubuntu/ eoan main restricted
|
||||
deb http://fi.archive.ubuntu.com/ubuntu/ eoan-updates main restricted
|
||||
deb http://fi.archive.ubuntu.com/ubuntu/ eoan universe
|
||||
deb http://fi.archive.ubuntu.com/ubuntu/ eoan-updates universe
|
||||
deb http://fi.archive.ubuntu.com/ubuntu/ eoan multiverse
|
||||
deb http://fi.archive.ubuntu.com/ubuntu/ eoan-updates multiverse
|
||||
deb http://fi.archive.ubuntu.com/ubuntu/ eoan-backports main restricted universe multiverse
|
||||
deb http://security.ubuntu.com/ubuntu/ eoan-security main restricted
|
||||
deb http://security.ubuntu.com/ubuntu/ eoan-security universe
|
||||
deb http://security.ubuntu.com/ubuntu/ eoan-security multiverse
|
||||
@@ -1,201 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
#===- clang-tidy-diff.py - ClangTidy Diff Checker ------------*- python -*--===#
|
||||
#
|
||||
# The LLVM Compiler Infrastructure
|
||||
#
|
||||
# This file is distributed under the University of Illinois Open Source
|
||||
# License. See LICENSE.TXT for details.
|
||||
#
|
||||
#===------------------------------------------------------------------------===#
|
||||
|
||||
r"""
|
||||
ClangTidy Diff Checker
|
||||
======================
|
||||
|
||||
This script reads input from a unified diff, runs clang-tidy on all changed
|
||||
files and outputs clang-tidy warnings in changed lines only. This is useful to
|
||||
detect clang-tidy regressions in the lines touched by a specific patch.
|
||||
Example usage for git/svn users:
|
||||
|
||||
git diff -U0 HEAD^ | clang-tidy-diff.py -p1
|
||||
svn diff --diff-cmd=diff -x-U0 | \
|
||||
clang-tidy-diff.py -fix -checks=-*,modernize-use-override
|
||||
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import multiprocessing
|
||||
import os
|
||||
import threading
|
||||
|
||||
is_py2 = sys.version[0] == '2'
|
||||
|
||||
if is_py2:
|
||||
import Queue as queue
|
||||
else:
|
||||
import queue as queue
|
||||
|
||||
def run_tidy(command, lines_by_file, queue, failed_files):
|
||||
"""Takes filenames out of queue and runs clang-tidy on them."""
|
||||
while True:
|
||||
name = queue.get()
|
||||
|
||||
line_filter_json = json.dumps([{"name" : name, "lines" : lines_by_file[name]}], separators = (',', ':'))
|
||||
if sys.platform == 'win32':
|
||||
line_filter_json = re.sub(r'"', r'"""', line_filter_json)
|
||||
else:
|
||||
line_filter_json = "'" + line_filter_json + "'";
|
||||
|
||||
invocation = list(command)
|
||||
invocation.append('-line-filter=' + line_filter_json)
|
||||
invocation.append(name)
|
||||
|
||||
sys.stdout.write('Checking differences in {}...\n'.format(name))
|
||||
return_code = subprocess.call(' '.join(invocation), shell=True)
|
||||
if return_code != 0:
|
||||
failed_files.append(name)
|
||||
queue.task_done()
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=
|
||||
'Run clang-tidy against changed files, and '
|
||||
'output diagnostics only for modified '
|
||||
'lines.')
|
||||
parser.add_argument('-clang-tidy-binary', metavar='PATH',
|
||||
default='clang-tidy',
|
||||
help='path to clang-tidy binary')
|
||||
parser.add_argument('-p', metavar='NUM', default=0,
|
||||
help='strip the smallest prefix containing P slashes')
|
||||
parser.add_argument('-regex', metavar='PATTERN', default=None,
|
||||
help='custom pattern selecting file paths to check '
|
||||
'(case sensitive, overrides -iregex)')
|
||||
parser.add_argument('-iregex', metavar='PATTERN', default=
|
||||
r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)',
|
||||
help='custom pattern selecting file paths to check '
|
||||
'(case insensitive, overridden by -regex)')
|
||||
|
||||
parser.add_argument('-fix', action='store_true', default=False,
|
||||
help='apply suggested fixes')
|
||||
parser.add_argument('-checks',
|
||||
help='checks filter, when not specified, use clang-tidy '
|
||||
'default',
|
||||
default='')
|
||||
parser.add_argument('-path', dest='build_path',
|
||||
help='Path used to read a compile command database.')
|
||||
parser.add_argument('-extra-arg', dest='extra_arg',
|
||||
action='append', default=[],
|
||||
help='Additional argument to append to the compiler '
|
||||
'command line.')
|
||||
parser.add_argument('-extra-arg-before', dest='extra_arg_before',
|
||||
action='append', default=[],
|
||||
help='Additional argument to prepend to the compiler '
|
||||
'command line.')
|
||||
parser.add_argument('-j', type=int, default=0,
|
||||
help='number of tidy instances to be run in parallel.')
|
||||
parser.add_argument('-quiet', action='store_true', default=False,
|
||||
help='Run clang-tidy in quiet mode')
|
||||
clang_tidy_args = []
|
||||
argv = sys.argv[1:]
|
||||
if '--' in argv:
|
||||
clang_tidy_args.extend(argv[argv.index('--'):])
|
||||
argv = argv[:argv.index('--')]
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
# Extract changed lines for each file.
|
||||
filename = None
|
||||
lines_by_file = {}
|
||||
for line in sys.stdin:
|
||||
match = re.search('^\+\+\+\ \"?(.*?/){%s}([^ \t\n\"]*)' % args.p, line)
|
||||
if match:
|
||||
filename = match.group(2)
|
||||
if filename == None:
|
||||
continue
|
||||
|
||||
if args.regex is not None:
|
||||
if not re.match('^%s$' % args.regex, filename):
|
||||
continue
|
||||
else:
|
||||
if not re.match('^%s$' % args.iregex, filename, re.IGNORECASE):
|
||||
continue
|
||||
|
||||
match = re.search('^@@.*\+(\d+)(,(\d+))?', line)
|
||||
if match:
|
||||
start_line = int(match.group(1))
|
||||
line_count = 1
|
||||
if match.group(3):
|
||||
line_count = int(match.group(3))
|
||||
if line_count == 0:
|
||||
continue
|
||||
end_line = start_line + line_count - 1;
|
||||
lines_by_file.setdefault(filename, []).append([start_line, end_line])
|
||||
|
||||
if len(lines_by_file) == 0:
|
||||
print("No relevant changes found.")
|
||||
sys.exit(0)
|
||||
|
||||
# Run clang-tidy on files containing changes.
|
||||
command = [args.clang_tidy_binary]
|
||||
if args.fix:
|
||||
command.append('-fix')
|
||||
if args.checks != '':
|
||||
command.append('-checks=' + quote + args.checks + quote)
|
||||
if args.quiet:
|
||||
command.append('-quiet')
|
||||
if args.build_path is not None:
|
||||
command.append('-p=%s' % args.build_path)
|
||||
for arg in args.extra_arg:
|
||||
command.append('-extra-arg=%s' % arg)
|
||||
for arg in args.extra_arg_before:
|
||||
command.append('-extra-arg-before=%s' % arg)
|
||||
command.extend(clang_tidy_args)
|
||||
|
||||
try:
|
||||
invocation = list(command)
|
||||
invocation.append('-')
|
||||
subprocess.check_call(invocation)
|
||||
except:
|
||||
print("Unable to run clang-tidy: {}".format(' '.join(command)))
|
||||
sys.exit(1)
|
||||
|
||||
max_task = args.j
|
||||
if max_task == 0:
|
||||
max_task = multiprocessing.cpu_count()
|
||||
|
||||
return_code = 0
|
||||
try:
|
||||
# Spin up a bunch of tidy-launching threads.
|
||||
task_queue = queue.Queue(max_task)
|
||||
# List of files with a non-zero return code.
|
||||
failed_files = []
|
||||
for _ in range(max_task):
|
||||
t = threading.Thread(target=run_tidy,
|
||||
args=(command, lines_by_file, task_queue, failed_files))
|
||||
t.daemon = True
|
||||
t.start()
|
||||
|
||||
# Fill the queue with files.
|
||||
for name in lines_by_file:
|
||||
task_queue.put(name)
|
||||
|
||||
# Wait for all threads to be done.
|
||||
task_queue.join()
|
||||
if len(failed_files):
|
||||
return_code = 1
|
||||
|
||||
except KeyboardInterrupt:
|
||||
# This is a sad hack. Unfortunately subprocess goes
|
||||
# bonkers with ctrl-c and we start forking merrily.
|
||||
print('\nCtrl-C detected, goodbye.')
|
||||
os.kill(0, 9)
|
||||
return_code = 1
|
||||
|
||||
sys.exit(return_code)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,31 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
CLANG_TIDY_PREFIX=${CLANG_TIDY_PREFIX:-$(scripts/mason.sh PREFIX clang-tidy VERSION 7.0.0)}
|
||||
CLANG_TIDY=${CLANG_TIDY_PREFIX}/bin/clang-tidy
|
||||
CLANG_APPLY=${CLANG_TIDY_PREFIX}/bin/clang-apply-replacements
|
||||
|
||||
for CLANG_FILE in "${CLANG_TIDY}" "${CLANG_APPLY}"; do
|
||||
command -v "${CLANG_FILE}" > /dev/null 2>&1 || {
|
||||
echo "Can't find ${CLANG_FILE} in PATH."
|
||||
if [ -z "${CLANG_FILE}" ]; then
|
||||
echo "Alternatively, you can manually set ${!CLANG_FILE@}."
|
||||
fi
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
echo "Running Clang checks... (this might take a while)"
|
||||
|
||||
if [[ -n ${3:-} ]] && [[ ${3:-} == "--diff" ]]; then
|
||||
git diff "origin/$2" | "scripts/clang-tidy-diff.py" \
|
||||
-clang-tidy-binary "${CLANG_TIDY}" \
|
||||
-p 1 -quiet -path "$1"
|
||||
else
|
||||
git ls-files "src/*.cpp" "test/*.cpp" "platform/*.cpp" "bin/*.cpp" | \
|
||||
xargs "${CLANG_TIDY_PREFIX}/share/run-clang-tidy.py" \
|
||||
-clang-tidy-binary "${CLANG_TIDY}" \
|
||||
-clang-apply-replacements-binary "${CLANG_APPLY}" \
|
||||
-quiet -p "$1" -fix
|
||||
fi
|
||||
@@ -1,38 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
#
|
||||
# This script takes three values: $1 should be a decimal value reflecting the
|
||||
# percentage of lines covered, with a maximum value of 100.0, $2 is
|
||||
# the platform the percentage pertains to (iOS or Android), and $3 is the
|
||||
# test scheme being run (on iOS, this is currently CI).
|
||||
#
|
||||
|
||||
# Check that platform input values are valid.
|
||||
|
||||
if [[ ! $2 = "iOS" && ! $2 = "Android" ]]; then
|
||||
echo "$2 does not match either 'iOS' or 'Android'. Platform must be specified for coverage report."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
circle_sha=""
|
||||
if [[ $CIRCLE_SHA1 ]]; then
|
||||
circle_sha="$CIRCLE_SHA1"
|
||||
fi
|
||||
# Create a formatted JSON file that contains the current coverage.
|
||||
|
||||
current_date=$(TZ=UTC date +"%FT%T%z")
|
||||
file_name=$2_coverage.json
|
||||
cat <<EOF > $file_name
|
||||
{"code_coverage":$1,"platform":"$2","sdk":"Maps","scheme":"$3","created_at":"$current_date","sha":"$circle_sha"}
|
||||
EOF
|
||||
gzip -f $file_name
|
||||
|
||||
if [ -z `which aws` ]; then
|
||||
brew install awscli
|
||||
fi
|
||||
|
||||
aws s3 cp $file_name.gz s3://mapbox-loading-dock/raw/mobile.codecoverage/$current_date/
|
||||
echo $
|
||||
@@ -1,211 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- mode: python; coding: utf-8 -*-
|
||||
|
||||
# All files in doxy-coverage are Copyright 2014 Alvaro Lopez Ortega.
|
||||
#
|
||||
# Authors:
|
||||
# * Alvaro Lopez Ortega <alvaro@gnu.org>
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions
|
||||
# are met:
|
||||
#
|
||||
# * Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
#
|
||||
# * Redistributions in binary form must reproduce the above
|
||||
# copyright notice, this list of conditions and the following
|
||||
# disclaimer in the documentation and/or other materials provided
|
||||
# with the distribution.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
__author__ = "Alvaro Lopez Ortega"
|
||||
__email__ = "alvaro@alobbs.com"
|
||||
__copyright__ = "Copyright (C) 2014 Alvaro Lopez Ortega"
|
||||
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
# Defaults
|
||||
ACCEPTABLE_COVERAGE = 80
|
||||
|
||||
# Global
|
||||
ns = None
|
||||
|
||||
def ERROR(*objs):
|
||||
print("ERROR: ", *objs, end='\n', file=sys.stderr)
|
||||
|
||||
def FATAL(*objs):
|
||||
ERROR (*objs)
|
||||
sys.exit((1,0)[ns.noerror])
|
||||
|
||||
def parse_file(fullpath):
|
||||
tree = ET.parse(fullpath)
|
||||
|
||||
sourcefile = None
|
||||
definitions = {}
|
||||
|
||||
for definition in tree.findall("./compounddef//memberdef"):
|
||||
# Should it be documented
|
||||
if definition.get('prot') == 'private' or (
|
||||
definition.get('kind') == 'function' and
|
||||
definition.get('static') == 'yes'):
|
||||
continue
|
||||
|
||||
# Is the definition documented?
|
||||
documented = False
|
||||
for k in ('briefdescription', 'detaileddescription', 'inbodydescription'):
|
||||
if definition.findall("./%s/"%(k)):
|
||||
documented = True
|
||||
break
|
||||
|
||||
# Name
|
||||
d_def = definition.find('./definition')
|
||||
d_nam = definition.find('./name')
|
||||
|
||||
if not sourcefile:
|
||||
l = definition.find('./location')
|
||||
if l is not None:
|
||||
sourcefile = l.get('file')
|
||||
|
||||
if d_def is not None:
|
||||
name = d_def.text
|
||||
elif d_nam is not None:
|
||||
name = d_nam.text
|
||||
else:
|
||||
name = definition.get('id')
|
||||
|
||||
# Aggregate
|
||||
definitions[name] = documented
|
||||
|
||||
if not sourcefile:
|
||||
sourcefile = fullpath
|
||||
|
||||
return (sourcefile, definitions)
|
||||
|
||||
|
||||
def parse(path):
|
||||
index_fp = os.path.join (path, "index.xml")
|
||||
if not os.path.exists (index_fp):
|
||||
FATAL ("Documentation not present. Exiting.", index_fp)
|
||||
|
||||
tree = ET.parse(index_fp)
|
||||
|
||||
files = {}
|
||||
for entry in tree.findall('compound'):
|
||||
if entry.get('kind') in ('dir'):
|
||||
continue
|
||||
|
||||
file_fp = os.path.join (path, "%s.xml" %(entry.get('refid')))
|
||||
tmp = parse_file (file_fp)
|
||||
files[tmp[0]] = tmp[1]
|
||||
|
||||
return files
|
||||
|
||||
|
||||
def report (files, exclude_dirs):
|
||||
def get_coverage (f):
|
||||
defs = files[f]
|
||||
if not defs:
|
||||
return 100
|
||||
|
||||
doc_yes = len([d for d in defs.values() if d])
|
||||
doc_no = len([d for d in defs.values() if not d])
|
||||
return (doc_yes * 100.0 / (doc_yes + doc_no))
|
||||
|
||||
def file_cmp (a,b):
|
||||
return cmp(get_coverage(a), get_coverage(b))
|
||||
|
||||
files_sorted = files.keys()
|
||||
files_sorted.sort(file_cmp)
|
||||
files_sorted.reverse()
|
||||
|
||||
total_yes = 0
|
||||
total_no = 0
|
||||
|
||||
for f in files_sorted:
|
||||
skip = False
|
||||
for exclude_dir in exclude_dirs:
|
||||
if exclude_dir in f:
|
||||
skip = True
|
||||
break
|
||||
if skip:
|
||||
continue
|
||||
|
||||
defs = files[f]
|
||||
if not defs:
|
||||
continue
|
||||
|
||||
doc_yes = len([d for d in defs.values() if d])
|
||||
doc_no = len([d for d in defs.values() if not d])
|
||||
doc_per = doc_yes * 100.0 / (doc_yes + doc_no)
|
||||
|
||||
total_yes += doc_yes
|
||||
total_no += doc_no
|
||||
|
||||
if not ns.quiet:
|
||||
print ('%3d%% - %s - (%d of %d)'%(doc_per, f, doc_yes, (doc_yes + doc_no)))
|
||||
|
||||
defs_sorted = defs.keys()
|
||||
defs_sorted.sort()
|
||||
for d in defs_sorted:
|
||||
if not defs[d]:
|
||||
print ("\t", d)
|
||||
|
||||
total_all = total_yes + total_no
|
||||
total_per = total_yes * 100 / total_all
|
||||
if not ns.quiet:
|
||||
print()
|
||||
print("%d%% API documentation coverage (%d out of %d)" %(total_per, total_yes, total_all))
|
||||
|
||||
if ns.json:
|
||||
with open(ns.json, "w") as f:
|
||||
f.write('{"documented": %d, "total": %d}\n' % (total_yes, total_all))
|
||||
|
||||
return (ns.threshold - total_per, 0)[total_per > ns.threshold]
|
||||
|
||||
|
||||
def main():
|
||||
# Arguments
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument ("dir", action="store", help="Path to Doxygen's XML doc directory")
|
||||
parser.add_argument ("--noerror", action="store_true", help="Do not return error code after execution")
|
||||
parser.add_argument ("--threshold", action="store", help="Min acceptable coverage percentage (Default: %s)"%(ACCEPTABLE_COVERAGE), default=ACCEPTABLE_COVERAGE, type=int)
|
||||
parser.add_argument ("--quiet", action="store_true", help="Don't output report to standard output")
|
||||
parser.add_argument ("--json", action="store", help="Output json coverage report to specified json file")
|
||||
parser.add_argument("--excludedirs", nargs='+', help="List of directories to be excluded from coverage analysis", type=str, default=[])
|
||||
|
||||
|
||||
global ns
|
||||
ns = parser.parse_args()
|
||||
if not ns:
|
||||
FATAL ("ERROR: Couldn't parse parameters")
|
||||
|
||||
# Parse
|
||||
files = parse (ns.dir)
|
||||
|
||||
# Print report
|
||||
err = report (files, ns.excludedirs)
|
||||
if ns.noerror:
|
||||
return
|
||||
|
||||
sys.exit(err)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
set -o pipefail
|
||||
set -u
|
||||
|
||||
SLACK_CHANNEL=${SLACK_CHANNEL:-''}
|
||||
SLACK_COLOR=${SLACK_COLOR:-''}
|
||||
|
||||
curl -g -H "Content-Type: application/json" -X POST \
|
||||
-d "{\"channel\": \"$SLACK_CHANNEL\", \"attachments\": [{\"text\": \"$SLACK_MESSAGE\", \"color\": \"$SLACK_COLOR\", \"author_name\": \"$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME\"}]}" \
|
||||
$SLACK_WEBHOOK_URL
|
||||
@@ -1,178 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const jwt = require('jsonwebtoken');
|
||||
const github = require('@octokit/rest').plugin(require('@octokit/plugin-retry'))({
|
||||
retry: { doNotRetry: [ /* Empty — retry on any error code. */ ] }
|
||||
})
|
||||
const zlib = require('zlib');
|
||||
const AWS = require('aws-sdk');
|
||||
|
||||
const SIZE_CHECK_APP_ID = 14028;
|
||||
const SIZE_CHECK_APP_INSTALLATION_ID = 229425;
|
||||
|
||||
process.on('unhandledRejection', error => {
|
||||
console.log(error);
|
||||
process.exit(1)
|
||||
});
|
||||
|
||||
const pk = process.env['SIZE_CHECK_APP_PRIVATE_KEY'];
|
||||
if (!pk) {
|
||||
console.log('Fork PR; not publishing size.');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const key = Buffer.from(pk, 'base64').toString('binary');
|
||||
const payload = {
|
||||
exp: Math.floor(Date.now() / 1000) + 60,
|
||||
iat: Math.floor(Date.now() / 1000),
|
||||
iss: SIZE_CHECK_APP_ID
|
||||
};
|
||||
|
||||
const token = jwt.sign(payload, key, {algorithm: 'RS256'});
|
||||
github.authenticate({type: 'app', token});
|
||||
|
||||
// Must be in sync with the definition in metrics/binary-size/index.html on the gh-pages branch.
|
||||
const platforms = [
|
||||
{ 'platform': 'iOS', 'arch': 'armv7' },
|
||||
{ 'platform': 'iOS', 'arch': 'arm64' },
|
||||
{ 'platform': 'iOS', 'arch': 'Dynamic' },
|
||||
{ 'platform': 'Android', 'arch': 'arm-v7' },
|
||||
{ 'platform': 'Android', 'arch': 'arm-v8' },
|
||||
{ 'platform': 'Android', 'arch': 'x86' },
|
||||
{ 'platform': 'Android', 'arch': 'x86_64' }
|
||||
];
|
||||
|
||||
const sizeCheckInfo = [];
|
||||
const rows = [];
|
||||
const date = new Date().toISOString().substring(0, 19);
|
||||
|
||||
function query(after) {
|
||||
return github.request({
|
||||
method: 'POST',
|
||||
url: '/graphql',
|
||||
headers: {
|
||||
// https://developer.github.com/changes/2018-07-11-graphql-checks-preview/
|
||||
accept: 'application/vnd.github.antiope-preview'
|
||||
},
|
||||
query: `query {
|
||||
repository(owner: "mapbox", name: "mapbox-gl-native") {
|
||||
ref(qualifiedName: "main") {
|
||||
target {
|
||||
... on Commit {
|
||||
history(first: 100, before: "36c6a8ea79bbd2596abb58ffb58debf65a4ea13d" ${after ? `, after: "${after}"` : ''}) {
|
||||
pageInfo {
|
||||
hasNextPage
|
||||
endCursor
|
||||
}
|
||||
edges {
|
||||
node {
|
||||
oid
|
||||
messageHeadline
|
||||
checkSuites(first: 1, filterBy: {appId: ${SIZE_CHECK_APP_ID}}) {
|
||||
nodes {
|
||||
checkRuns(first: 10) {
|
||||
nodes {
|
||||
name
|
||||
conclusion
|
||||
title
|
||||
summary
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}`
|
||||
}).then((result) => {
|
||||
|
||||
const history = result.data.data.repository.ref.target.history;
|
||||
|
||||
for (const edge of history.edges) {
|
||||
const commit = edge.node;
|
||||
const suite = commit.checkSuites.nodes[0];
|
||||
|
||||
if (!suite)
|
||||
continue;
|
||||
const allRuns = commit.checkSuites.nodes[0].checkRuns.nodes;
|
||||
|
||||
const sizeCheckRuns = allRuns.filter(function (run) {
|
||||
return run.name.match(/Size - (\w+) ([\w-]+)/);
|
||||
});
|
||||
|
||||
const row = [`${commit.oid.slice(0, 7)} - ${commit.messageHeadline}`];
|
||||
|
||||
for (let i = 0; i < platforms.length; i++) {
|
||||
const {platform, arch} = platforms[i];
|
||||
|
||||
const run = sizeCheckRuns.find((run) => {
|
||||
const [, p, a] = run.name.match(/Size - (\w+) ([\w-]+)/);
|
||||
return platform === p && arch === a;
|
||||
});
|
||||
|
||||
row[i + 1] = run ? +run.summary.match(/is (\d+) bytes/)[1] : undefined;
|
||||
}
|
||||
rows.push(row);
|
||||
}
|
||||
|
||||
if (history.pageInfo.hasNextPage) {
|
||||
return query(history.pageInfo.endCursor);
|
||||
} else {
|
||||
const latestRun = rows[0]
|
||||
const runSizeMeasurements = latestRun.slice(1)
|
||||
|
||||
for (let i = 0; i < platforms.length; i++) {
|
||||
const {platform, arch} = platforms[i];
|
||||
|
||||
sizeCheckInfo.push(JSON.stringify({
|
||||
'sdk': 'maps',
|
||||
'platform' : platform,
|
||||
'arch': arch,
|
||||
'size' : runSizeMeasurements[i],
|
||||
'created_at': date
|
||||
}));
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
github.apps.createInstallationToken({installation_id: SIZE_CHECK_APP_INSTALLATION_ID})
|
||||
.then(({data}) => {
|
||||
github.authenticate({type: 'token', token: data.token});
|
||||
|
||||
return query().then(function() {
|
||||
// Uploads to data source used by
|
||||
// http://mapbox.github.io/mapbox-gl-native/metrics/binary-size/
|
||||
var firstPromise = new AWS.S3({region: 'us-east-1'}).putObject({
|
||||
Body: zlib.gzipSync(JSON.stringify(rows.reverse())),
|
||||
Bucket: 'mapbox',
|
||||
Key: 'mapbox-gl-native/metrics/binary-size/data.json',
|
||||
ACL: 'public-read',
|
||||
CacheControl: 'max-age=300',
|
||||
ContentEncoding: 'gzip',
|
||||
ContentType: 'application/json'
|
||||
}).promise();
|
||||
// Uploads to data source used by Mapbox internal metrics dashboards
|
||||
var secondPromise = new AWS.S3({region: 'us-east-1'}).putObject({
|
||||
Body: zlib.gzipSync(sizeCheckInfo.join('\n')),
|
||||
Bucket: 'mapbox-loading-dock',
|
||||
Key: `raw/mobile.binarysize/${date.substring(0,10)}/mapbox-maps-ios-android-${process.env['CIRCLE_SHA1']}.json.gz`,
|
||||
CacheControl: 'max-age=300',
|
||||
ContentEncoding: 'gzip',
|
||||
ContentType: 'application/json'
|
||||
}).promise();
|
||||
|
||||
return Promise.all([firstPromise, secondPromise]).then(data => {
|
||||
return console.log("Successfully uploaded all binary size metrics to S3");
|
||||
}).catch(err => {
|
||||
console.log("Error uploading binary size metrics to S3 " + err.message);
|
||||
return err;
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -1,170 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
// Script to retrieve total code coverage ratio from codecov.io
|
||||
// for a given commit hash, and upload it to the S3 bucket.
|
||||
|
||||
const https = require('https');
|
||||
const zlib = require('zlib');
|
||||
const AWS = require('aws-sdk');
|
||||
const {execSync} = require('child_process');
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const options = {
|
||||
help: false
|
||||
};
|
||||
|
||||
const usage = 'usage: publish_code_coverage.js [options]\n' +
|
||||
'options: \n' +
|
||||
' -h, --help \n' +
|
||||
' -p, --platform <Platform Name>\n' +
|
||||
' -s, --sdk <Sdk Name> \n' +
|
||||
' -c, --commit <Commit Hash> \n';
|
||||
|
||||
for (var i = 0; i < args.length; i++) {
|
||||
var arg = args[i];
|
||||
|
||||
switch(arg) {
|
||||
case '-h':
|
||||
case '--help':
|
||||
options.help = true;
|
||||
break;
|
||||
case '-s':
|
||||
case '--sdk':
|
||||
options.sdkName = args[i + 1];
|
||||
break;
|
||||
case '-p':
|
||||
case '--platform':
|
||||
options.platformName = args[i + 1];
|
||||
break;
|
||||
case '-c':
|
||||
case '--commit':
|
||||
options.commitId = args[i + 1];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (options.help == true) {
|
||||
console.log(usage);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Commit hash
|
||||
const commitHash = options.commitId ? options.commitId : process.env['CIRCLE_SHA1'];
|
||||
if (!options.sdkName || !options.platformName || !commitHash) {
|
||||
console.log(usage);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Commit Message
|
||||
const commitMessage = execSync(`git show --pretty=format:%s -s ${commitHash}`).toString().trim();
|
||||
if (!commitMessage) {
|
||||
throw new Error ('Commit message is missing');
|
||||
}
|
||||
|
||||
const date = new Date().toISOString().substring(0, 19);
|
||||
|
||||
process.on('uncaughtException', (err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
// Parse the response received from codecov.io and build the
|
||||
// data point that is going to be uploaded to S3 bucket.
|
||||
function parseResponse(data) {
|
||||
if (data && data.commit) {
|
||||
if (!data.commit.totals || !data.commit.totals.c) {
|
||||
return;
|
||||
}
|
||||
|
||||
const source = {
|
||||
code_coverage: Number(data.commit.totals.c),
|
||||
platform: options.platformName,
|
||||
sdk: options.sdkName,
|
||||
commit: commitHash,
|
||||
commit_message: commitMessage,
|
||||
created_at: date
|
||||
};
|
||||
|
||||
return source;
|
||||
}
|
||||
}
|
||||
|
||||
// Upload to data source used by Mapbox internal metrics dashboards
|
||||
function uploadData(data) {
|
||||
return new AWS.S3({region: 'us-east-1'}).putObject({
|
||||
Body: zlib.gzipSync(JSON.stringify(data)),
|
||||
Bucket: 'mapbox-loading-dock',
|
||||
Key: `raw/mobile_staging.codecoverage/${date.substring(0,10)}/${options.sdkName}-coverage-${commitHash}.json.gz`,
|
||||
CacheControl: 'max-age=300',
|
||||
ContentEncoding: 'gzip',
|
||||
ContentType: 'application/json'
|
||||
}).promise();
|
||||
}
|
||||
|
||||
// Attempt to retrieve code coverage report from codecov.io
|
||||
// for a given commit hash.
|
||||
function httpRequest() {
|
||||
const options = {
|
||||
hostname: 'codecov.io',
|
||||
port: 443,
|
||||
path: '/api/gh/mapbox/mapbox-gl-native/commit/' + commitHash,
|
||||
method: 'GET'
|
||||
};
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
setTimeout(function() {
|
||||
const req = https.request(options, (res) => {
|
||||
var body = [];
|
||||
res.on('data', (chunk) => {
|
||||
body.push(chunk);
|
||||
}).on('error', (error) => {
|
||||
reject(error);
|
||||
}).on('end', () => {
|
||||
if (res.statusCode < 200 || res.statusCode >= 300) {
|
||||
return reject(new Error('Failed to fetch coverage report from codecov.io. StatusCode=' + res.statusCode));
|
||||
}
|
||||
|
||||
try {
|
||||
body = JSON.parse(Buffer.concat(body).toString());
|
||||
resolve(body);
|
||||
} catch(e) {
|
||||
reject(e);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Reject on error
|
||||
req.on('error', (err) => {
|
||||
reject(err);
|
||||
});
|
||||
|
||||
req.end();
|
||||
}, 30000);
|
||||
});
|
||||
}
|
||||
|
||||
var errResponse = false;
|
||||
const publishWithRetry = (maxRetries) => {
|
||||
httpRequest().then((body) => {
|
||||
const dataSource = parseResponse(body);
|
||||
if (dataSource) {
|
||||
return uploadData(dataSource);
|
||||
} else {
|
||||
errResponse = true;
|
||||
throw new Error('Failed to parse coverage report received from codecov.io.');
|
||||
}
|
||||
}).then(data => {
|
||||
console.log('Successfully uploaded code coverage metrics to S3');
|
||||
}).catch(err => {
|
||||
if (maxRetries > 1 && errResponse) {
|
||||
console.log('Invalid coverage report received. Trying to retrieve again.');
|
||||
errResponse = false;
|
||||
return publishWithRetry(maxRetries - 1);
|
||||
}
|
||||
|
||||
console.error('Failed to upload code coverage metrics to S3: ' + err.message);
|
||||
});
|
||||
};
|
||||
|
||||
// Fetch and publish code coverage report
|
||||
publishWithRetry(5);
|
||||
@@ -1,92 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const jwt = require('jsonwebtoken');
|
||||
const github = require('@octokit/rest').plugin(require('@octokit/plugin-retry'))({
|
||||
retry: { doNotRetry: [ /* Empty — retry on any error code. */ ] }
|
||||
})
|
||||
const zlib = require('zlib');
|
||||
const AWS = require('aws-sdk');
|
||||
const fs = require('fs');
|
||||
|
||||
const SIZE_CHECK_APP_ID = 14028;
|
||||
const SIZE_CHECK_APP_INSTALLATION_ID = 229425;
|
||||
|
||||
var coverage;
|
||||
|
||||
try {
|
||||
coverage = JSON.parse(fs.readFileSync(process.argv[2]));
|
||||
if (typeof coverage.documented !== 'number' || typeof coverage.total !== 'number' ||
|
||||
coverage.documented > coverage.total) {
|
||||
throw new Error('Invalid coverage.json file');
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('Failed to parse json file with coverage data: ', error);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
process.on('unhandledRejection', error => {
|
||||
console.log(error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
const pk = process.env['SIZE_CHECK_APP_PRIVATE_KEY'];
|
||||
if (!pk) {
|
||||
console.log('Fork PR; not publishing size.');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const key = Buffer.from(pk, 'base64').toString('binary');
|
||||
const payload = {
|
||||
exp: Math.floor(Date.now() / 1000) + 60,
|
||||
iat: Math.floor(Date.now() / 1000),
|
||||
iss: SIZE_CHECK_APP_ID
|
||||
};
|
||||
|
||||
const token = jwt.sign(payload, key, {algorithm: 'RS256'});
|
||||
github.authenticate({type: 'app', token});
|
||||
|
||||
github.apps.createInstallationToken({installation_id: SIZE_CHECK_APP_INSTALLATION_ID})
|
||||
.then(({data}) => {
|
||||
github.authenticate({type: 'token', token: data.token});
|
||||
const percentage = coverage.documented * 100 / coverage.total;
|
||||
const title = `${percentage.toFixed(2)}%`;
|
||||
const date = new Date().toISOString().substring(0, 19);
|
||||
|
||||
var promises = [github.checks.create({
|
||||
owner: 'mapbox',
|
||||
repo: 'mapbox-gl-native',
|
||||
name: 'Doxygen coverage',
|
||||
head_branch: process.env['CIRCLE_BRANCH'],
|
||||
head_sha: process.env['CIRCLE_SHA1'],
|
||||
status: 'completed',
|
||||
conclusion: 'success',
|
||||
completed_at: new Date().toISOString(),
|
||||
output: {
|
||||
title,
|
||||
summary: `There is doxygen documentation for ${percentage}% of the public symbols (${coverage.documented} out of ${coverage.total})`
|
||||
}
|
||||
})];
|
||||
|
||||
if (process.env['CIRCLE_BRANCH'] === 'main') {
|
||||
promises.push(new AWS.S3({region: 'us-east-1'}).putObject({
|
||||
Body: zlib.gzipSync(JSON.stringify({
|
||||
'created_at': date,
|
||||
'documented': coverage.documented,
|
||||
'total': coverage.total,
|
||||
'commit': process.env['CIRCLE_SHA1']
|
||||
})),
|
||||
Bucket: 'mapbox-loading-dock',
|
||||
Key: `raw/mobile_staging.docs_coverage/${date.substring(0,10)}/${process.env['CIRCLE_SHA1']}.json.gz`,
|
||||
CacheControl: 'max-age=300',
|
||||
ContentEncoding: 'gzip',
|
||||
ContentType: 'application/json'
|
||||
}).promise());
|
||||
}
|
||||
|
||||
return Promise.all(promises).then(data => {
|
||||
return console.log('Succesfully uploaded doxygen coverage metrics');
|
||||
}).catch(err => {
|
||||
console.log('Error uploading doxygen coverage metrics: ' + err.message);
|
||||
return err;
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user