Re: [PATCH/RFC] tests: WIP Infrastructure for Git smoke testing

Ævar Arnfjörð Bjarmason wrote:

BTW, below is the script I have devised to automatically run the tests under valgrind and then bisect any offenders. (It keeps exposing odd test failures and stealing my time tracking them down...)

It gives fairly good results, but since it's also rather kludgy I want to test and improve it a bit more before shooting for contrib (if anything).

Note that running this will take on the order of hours at least.

---- 8< ---- #!/bin/sh

MAILTO=tr@thomasrast.ch

branch="${1:-next}" logfile="$(pwd)/bisection-log-$$"

export DIFF=diff # stupid bug

die () {
	(echo "Error message: $*"; echo '-----'; cat "$logfile") |
	mail -s "Testing $branch: die() called!" "$MAILTO"
	echo "fatal: $*" >&2
	exit 1
}

git fetch origin 2>&1 | tee "$logfile" || die "fetch failed"

# test if we've already done this test "$(git rev-parse last_known_good_$branch)" = "$(git rev-parse origin/$branch)" && exit

git checkout -f origin/"$branch" 2>&1 | tee "$logfile" || die "checkout failed (can't happen)"

results="test-results.$(git describe)" mkdir "$results"

make -j12 2>&1 | tee -a "$logfile" || die "initial compilation failed?" make -k -j8 test 2>&1 | tee -a "$logfile"

if [ ! -d t/"test-results" ]; then
    mail -s "Valgrind-tested $branch: all good!" "$MAILTO" < "$logfile"
    git tag -f last_known_good_$branch origin/$branch
    rm "$logfile"
    exit
fi

cd t/ cp -a test-results "$results" ( cd $results && perl -i -ne 'print unless /^==.*execve/i' *.out ) failing_normally=$(cd "$results" && grep -L '^0$' *.exit) failing_valgrind=$(cd "$results" && grep -lE '^==[0-9]+==' *.out) cd ..

echo "failed (normal):" $failing_normally | tee -a "$logfile" echo "failed (valgrind):" $failing_valgrind | tee -a "$logfile"

if [ -z "$failing_normally" -a -z "$failing_valgrind" ]; then
    mail -s "Valgrind-tested $branch: all good!" "$MAILTO" < "$logfile"
    git tag -f last_known_good_$branch origin/$branch
    rm "$logfile"
    exit
fi

mail -s "Valgrind-testing $branch: starting test bisection" "$MAILTO" < "$logfile"

for test_out in $failing_valgrind; do
    test="${test_out%.out}"
    echo "bisecting valgrind-failing test $test"
    git bisect start > t/"$results"/$test.bisect 2>&1
    git bisect good last_known_good_pu last_known_good_next 2>&1 | tee t/"$results"/$test.bisect
    git bisect bad origin/$branch 2>&1 | tee -a t/"$results"/$test.bisect
    git bisect run sh -c "test ! -f t/$test.sh || { make -j12 && cd t && ./$test.sh --valgrind --tee -i && ! grep -E '^==[0-9]+==' test-results/$test.out | grep -vi execve; }" 2>&1 | tee -a t/"$results"/$test.bisect
    git bisect log 2>&1 | tee -a t/"$results"/$test.bisect
    git bisect reset
    mail -s "Bisection results for $test (valgrind)" "$MAILTO" < t/"$results"/$test.bisect
done

for test_out in $failing_normally; do
    test="${test_out%.exit}"
    case "$failing_valgrind" in
	*$test.out*)
	    continue
	    ;;
    esac
    echo "bisecting failing test $test"
    git bisect start 2>&1 | tee t/"$results"/$test.bisect 2>&1
    git bisect good last_known_good_pu last_known_good_pu 2>&1 | tee -a t/"$results"/$test.bisect 2>&1
    git bisect bad origin/$branch 2>&1 | tee -a t/"$results"/$test.bisect 2>&1
    git bisect run sh -c "test ! -f t/$test.sh || { make -j12 && cd t && ./$test.sh --tee -i; }" 2>&1 | tee -a t/"$results"/$test.bisect 2>&1
    git bisect log | tee -a t/"$results"/$test.bisect
    git bisect reset
    mail -s "Bisection results for $test" "$MAILTO" < t/"$results"/$test.bisect
done

rm "$logfile" ---- >8 ----