42 lines
1.2 KiB
Bash
Executable File
42 lines
1.2 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Usage:
|
|
# check_benchmark_results.sh <baseline> <results> <comparison report in txt> <comparison report in html>
|
|
|
|
# Benchmark comparison script uses a hard-coded 5% threshold by default, but it causes random failures
|
|
# Increase the threshold to 15%
|
|
|
|
if [ "$(uname)" == "Darwin" ]; then
|
|
sed -i '' s/"if res > 0.05"/"if res > 0.15"/ $PWD/vendor/benchmark/tools/gbench/report.py
|
|
elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
|
|
sed -i s/"if res > 0.05"/"if res > 0.15"/ $PWD/vendor/benchmark/tools/gbench/report.py
|
|
fi
|
|
|
|
# Run the bencmark comparison
|
|
vendor/benchmark/tools/compare.py benchmarks $1 $2 > $3
|
|
|
|
# Find cases where the average is above threshold
|
|
tests=`fgrep mean $3 | cut -c1-85 | grep 91m`
|
|
|
|
if [ -z "$tests" ]
|
|
then
|
|
echo "" >> $3
|
|
echo "PASSED, all benchmarks within 15% threshold" >> $3
|
|
|
|
ansi2html < $3 > $4
|
|
exit 0
|
|
else
|
|
test_names=`fgrep mean $3 | cut -c1-85 | grep 91m | cut -f1 -d' '`
|
|
|
|
echo "" >> $3
|
|
echo "FAILED, following benchmarks are not within 15% threshold:" >> $3
|
|
echo "" >> $3
|
|
for t in $test_names;
|
|
do
|
|
echo $t >> $3
|
|
done
|
|
|
|
ansi2html < $3 > $4
|
|
exit 1
|
|
fi
|