diff options
| -rw-r--r-- | .gitignore | 3 | ||||
| -rwxr-xr-x | util/romcc/test.sh | 232 |
2 files changed, 233 insertions, 2 deletions
diff --git a/.gitignore b/.gitignore index 912aa85eb2..07a48358a3 100644 --- a/.gitignore +++ b/.gitignore @@ -96,8 +96,7 @@ util/nvramtool/.dependencies util/nvramtool/nvramtool util/optionlist/Options.wiki util/romcc/romcc -util/romcc/tests/fail_test*.S -util/romcc/tests/*.S-O2-mmmx +util/romcc/build util/runfw/googlesnow util/superiotool/superiotool util/vgabios/testbios diff --git a/util/romcc/test.sh b/util/romcc/test.sh new file mode 100755 index 0000000000..8aa7c53a79 --- /dev/null +++ b/util/romcc/test.sh @@ -0,0 +1,232 @@ +#!/bin/sh +# +# This file is part of the coreboot project. +# +# Copyright 2016 Jonathan Neuschäfer <j.neuschaefer@gmx.net> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + + +# These tests are currently known to be broken and should not result in a +# build failure: +XFAIL_TESTS=" + simple_test4.c + simple_test6.c + simple_test25.c + simple_test26.c + simple_test46.c + simple_test47.c + simple_test54.c + simple_test72.c + simple_test73.c + linux_test2.c + linux_test5.c + linux_test10.c + linux_test11.c + linux_test12.c +" + +# ------------------------------------------------------------------- # + +BASEDIR="$(dirname "$0")" +BUILDDIR="$BASEDIR/build" +LOGDIR="$BUILDDIR/logs" +mkdir -p "$BUILDDIR" +mkdir -p "$LOGDIR" + +usage () { + echo "Usage: test.sh [--nocolor] CLASS" + echo "" + echo "CLASS selects a group of tests to run. It must be one of the following:" + echo " all - all tests" + echo " simple - simple tests" + echo " linux - linux programs whose output is checked against a reference" + echo "" + echo "--nocolor disables colors." + exit 1 +} + +COLORS=1 +if [ "$1" = "--nocolor" ]; then + shift + COLORS=0 +fi + + +if [ -t 1 -a "$COLORS" -eq 1 ]; then + red() { printf "\033[1;31m%s\033[0m" "$*"; } + green() { printf "\033[1;32m%s\033[0m" "$*"; } + blue() { printf "\033[1;34m%s\033[0m" "$*"; } +else + red() { printf "%s" "$*"; } + green() { printf "%s" "$*"; } + blue() { printf "%s" "$*"; } +fi + +init_stats() { + NUM_TOTAL=0 # Number of tests that were run + NUM_FAIL=0 # Number of tests that failed unexpectedly + NUM_BROKEN=0 # Number of tests that failed expectedly + NUM_PASS=0 # Number of tests that passed expectedly + NUM_FIXED=0 # Number of tests that passed unexpectedly +} + +get_romcc() { + ROMCC="$BASEDIR/romcc" + if [ ! -f "$ROMCC" ]; then + echo "romcc not found! Please run \"make romcc\"." + exit 1 + fi +} + +init_testing() { + init_stats + get_romcc +} + +show_stats() { + printf "passed: %s\t(%s newly fixed)\n" $NUM_PASS $NUM_FIXED + printf "failed: %s\t(%s known broken)\n" $NUM_FAIL $NUM_BROKEN + printf "total: %s\n" $NUM_TOTAL +} + +is_xfail() { + local t + for t in $XFAIL_TESTS; do + if [ "$t" = "$1" ]; then + return 0 + fi + done + return 1 +} + +pass() { + NUM_TOTAL=$((NUM_TOTAL + 1)) + NUM_PASS=$((NUM_PASS + 1)) + + green "passed" + if is_xfail "$(basename "$1")"; then + blue " (fixed)" + NUM_FIXED=$((NUM_FIXED + 1)) + fi + echo +} + +fail() { + NUM_TOTAL=$((NUM_TOTAL + 1)) + NUM_FAIL=$((NUM_FAIL + 1)) + + red "failed" + if is_xfail "$(basename "$1")"; then + blue " (known broken)" + NUM_BROKEN=$((NUM_BROKEN + 1)) + fi + echo +} + +run_simple_test() { + # TODO: "timeout" is not POSIX compliant. Use something that is. + timeout 60 "$ROMCC" $1 "$2" -o "$BUILDDIR/dummy.S" +} + +run_simple_tests() { + echo "Running simple tests..." + + local t + for t in $(find "$BASEDIR/tests" -name 'simple_test*.c'); do + printf "%s" "$(basename "$t")" + + local result=pass + local logfile="$LOGDIR/$(basename "$t").log" + rm "$logfile" >/dev/null 2>&1 + for opt in "" "-O" "-O2" "-mmmx" "-msse" "-mmmx -msse" \ + "-O -mmmx" "-O -msse" "-O -mmmx -msse" \ + "-O2 -mmmx" "-O2 -msse" "-O2 -mmmx -msse"; do + if run_simple_test "$opt" "$t" \ + >> "$logfile" 2>&1; then + printf . + else + result=fail + break + fi + done + printf " " + $result "$t" + done + + echo +} + +run_linux_test() { + local base="$(basename "$1")" + + # TODO: "timeout" is not POSIX compliant. Use something that is. + + timeout 60 "$ROMCC" "$1" -o "$BUILDDIR/$base.S" || return 1 + as --32 "$BUILDDIR/$base.S" -o "$BUILDDIR/$base.o" || return 1 + ld -m elf_i386 -T "$BASEDIR/tests/ldscript.ld" \ + "$BUILDDIR/$base.o" -o "$BUILDDIR/$base.elf" || return 1 + timeout 60 "$BUILDDIR/$base.elf" > "$BUILDDIR/$base.out" || return 1 + + diff -u "$BASEDIR/results/${base%.c}.out" "$BUILDDIR/$base.out" +} + +run_linux_tests() { + echo "Running linux tests..." + + local t + for t in $(find "$BASEDIR/tests" -name 'linux_test*.c'); do + printf "%s... " "$(basename "$t")" + + local logfile="$LOGDIR/$(basename "$t").log" + if run_linux_test "$t" > "$logfile" 2>&1; then + pass "$t" + else + fail "$t" + fi + done + + echo +} + + +if [ $# -ne 1 ]; then + usage +fi + +CLASS="$1" + +case "$CLASS" in + all) + init_testing + run_simple_tests + run_linux_tests + show_stats + ;; + simple) + init_testing + run_simple_tests + show_stats + ;; + linux) + init_testing + run_linux_tests + show_stats + ;; + *) + echo "Invalid test class $CLASS" + echo + usage + ;; +esac + +if [ $NUM_FAIL -ne $NUM_BROKEN ]; then + exit 1 +fi |