#!/bin/sh # # This file is part of the coreboot project. # # Copyright 2016 Jonathan Neuschäfer <j.neuschaefer@gmx.net> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # These tests are currently known to be broken and should not result in a # build failure: XFAIL_TESTS=" simple_test4.c simple_test6.c simple_test25.c simple_test26.c simple_test46.c simple_test47.c simple_test54.c simple_test72.c simple_test73.c linux_test2.c linux_test5.c linux_test10.c linux_test11.c linux_test12.c " # ------------------------------------------------------------------- # BASEDIR="$(dirname "$0")" BUILDDIR="$BASEDIR/build" LOGDIR="$BUILDDIR/logs" mkdir -p "$BUILDDIR" mkdir -p "$LOGDIR" usage () { echo "Usage: test.sh [--nocolor] CLASS" echo "" echo "CLASS selects a group of tests to run. It must be one of the following:" echo " all - all tests" echo " simple - simple tests" echo " linux - linux programs whose output is checked against a reference" echo "" echo "--nocolor disables colors." exit 1 } COLORS=1 if [ "$1" = "--nocolor" ]; then shift COLORS=0 fi if [ -t 1 -a "$COLORS" -eq 1 ]; then red() { printf "\033[1;31m%s\033[0m" "$*"; } green() { printf "\033[1;32m%s\033[0m" "$*"; } blue() { printf "\033[1;34m%s\033[0m" "$*"; } else red() { printf "%s" "$*"; } green() { printf "%s" "$*"; } blue() { printf "%s" "$*"; } fi init_stats() { NUM_TOTAL=0 # Number of tests that were run NUM_FAIL=0 # Number of tests that failed unexpectedly NUM_BROKEN=0 # Number of tests that failed expectedly NUM_PASS=0 # Number of tests that passed expectedly NUM_FIXED=0 # Number of tests that passed unexpectedly } get_romcc() { ROMCC="$BUILDDIR/romcc" if [ ! -f "$ROMCC" ]; then echo "romcc not found! Please run \"make\"." exit 1 fi } init_testing() { init_stats get_romcc } show_stats() { printf "passed: %s\t(%s newly fixed)\n" $NUM_PASS $NUM_FIXED printf "failed: %s\t(%s known broken)\n" $NUM_FAIL $NUM_BROKEN printf "total: %s\n" $NUM_TOTAL } is_xfail() { local t for t in $XFAIL_TESTS; do if [ "$t" = "$1" ]; then return 0 fi done return 1 } pass() { NUM_TOTAL=$((NUM_TOTAL + 1)) NUM_PASS=$((NUM_PASS + 1)) green "passed" if is_xfail "$(basename "$1")"; then blue " (fixed)" NUM_FIXED=$((NUM_FIXED + 1)) fi echo } fail() { NUM_TOTAL=$((NUM_TOTAL + 1)) NUM_FAIL=$((NUM_FAIL + 1)) red "failed" if is_xfail "$(basename "$1")"; then blue " (known broken)" NUM_BROKEN=$((NUM_BROKEN + 1)) fi echo } run_simple_test() { # TODO: "timeout" is not POSIX compliant. Use something that is. timeout 60 "$ROMCC" $1 "$2" -o "$BUILDDIR/dummy.S" } run_simple_tests() { echo "Running simple tests..." local t for t in $(find "$BASEDIR/tests" -name 'simple_test*.c'); do printf "%s" "$(basename "$t")" local result=pass local logfile="$LOGDIR/$(basename "$t").log" rm "$logfile" >/dev/null 2>&1 for opt in "" "-O" "-O2" "-mmmx" "-msse" "-mmmx -msse" \ "-O -mmmx" "-O -msse" "-O -mmmx -msse" \ "-O2 -mmmx" "-O2 -msse" "-O2 -mmmx -msse"; do if run_simple_test "$opt" "$t" \ >> "$logfile" 2>&1; then printf . else result=fail break fi done printf " " $result "$t" done echo } run_linux_test() { local base="$(basename "$1")" # TODO: "timeout" is not POSIX compliant. Use something that is. timeout 60 "$ROMCC" "$1" -o "$BUILDDIR/$base.S" || return 1 as --32 "$BUILDDIR/$base.S" -o "$BUILDDIR/$base.o" || return 1 ld -m elf_i386 -T "$BASEDIR/tests/ldscript.ld" \ "$BUILDDIR/$base.o" -o "$BUILDDIR/$base.elf" || return 1 timeout 60 "$BUILDDIR/$base.elf" > "$BUILDDIR/$base.out" || return 1 diff -u "$BASEDIR/results/${base%.c}.out" "$BUILDDIR/$base.out" } run_linux_tests() { echo "Running linux tests..." local t for t in $(find "$BASEDIR/tests" -name 'linux_test*.c'); do printf "%s... " "$(basename "$t")" local logfile="$LOGDIR/$(basename "$t").log" if run_linux_test "$t" > "$logfile" 2>&1; then pass "$t" else fail "$t" fi done echo } if [ $# -ne 1 ]; then usage fi CLASS="$1" case "$CLASS" in all) init_testing run_simple_tests run_linux_tests show_stats ;; simple) init_testing run_simple_tests show_stats ;; linux) init_testing run_linux_tests show_stats ;; *) echo "Invalid test class $CLASS" echo usage ;; esac if [ $NUM_FAIL -ne $NUM_BROKEN ]; then exit 1 fi