From e69d6c2e7bef9e8821423cb476d1a727b73ca1a4 Mon Sep 17 00:00:00 2001 From: Jonathan Neuschäfer Date: Wed, 13 Apr 2016 02:47:24 +0200 Subject: romcc: Rewrite the test system MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Differences: - The test logic is now only implemented in one place (pending the deletion of the old parts), whereas it previously was implemented both as make rules and as a pair of shell scripts. - Tests don't need to be registered anymore. Just adding a new file with the correct name is enough to have it tested. - The code is hopefully more readable and maintainable. - The new test script supports colors (if the standard output is a terminal and --nocolor was not passed on the command line). Things to do in follow-up patches: - Remove the old test code - Test or remove fail_test*.c, hello_world*.c and raminit_test*.c - Fix regressions that have built up over the years, while making sure not to introduce new ones - Makefile integration - Jenkins integration There are tests in the makefile that specify -fno-always-inline, but this option doesn't exist anymore, so I didn't port them over. Change-Id: Idd6b89368c1e36555cb880c37bbe07035c938cd7 Signed-off-by: Jonathan Neuschäfer Reviewed-on: https://review.coreboot.org/14291 Reviewed-by: Martin Roth Tested-by: build bot (Jenkins) --- util/romcc/test.sh | 232 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100755 util/romcc/test.sh (limited to 'util/romcc/test.sh') diff --git a/util/romcc/test.sh b/util/romcc/test.sh new file mode 100755 index 0000000000..8aa7c53a79 --- /dev/null +++ b/util/romcc/test.sh @@ -0,0 +1,232 @@ +#!/bin/sh +# +# This file is part of the coreboot project. +# +# Copyright 2016 Jonathan Neuschäfer +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + + +# These tests are currently known to be broken and should not result in a +# build failure: +XFAIL_TESTS=" + simple_test4.c + simple_test6.c + simple_test25.c + simple_test26.c + simple_test46.c + simple_test47.c + simple_test54.c + simple_test72.c + simple_test73.c + linux_test2.c + linux_test5.c + linux_test10.c + linux_test11.c + linux_test12.c +" + +# ------------------------------------------------------------------- # + +BASEDIR="$(dirname "$0")" +BUILDDIR="$BASEDIR/build" +LOGDIR="$BUILDDIR/logs" +mkdir -p "$BUILDDIR" +mkdir -p "$LOGDIR" + +usage () { + echo "Usage: test.sh [--nocolor] CLASS" + echo "" + echo "CLASS selects a group of tests to run. It must be one of the following:" + echo " all - all tests" + echo " simple - simple tests" + echo " linux - linux programs whose output is checked against a reference" + echo "" + echo "--nocolor disables colors." + exit 1 +} + +COLORS=1 +if [ "$1" = "--nocolor" ]; then + shift + COLORS=0 +fi + + +if [ -t 1 -a "$COLORS" -eq 1 ]; then + red() { printf "\033[1;31m%s\033[0m" "$*"; } + green() { printf "\033[1;32m%s\033[0m" "$*"; } + blue() { printf "\033[1;34m%s\033[0m" "$*"; } +else + red() { printf "%s" "$*"; } + green() { printf "%s" "$*"; } + blue() { printf "%s" "$*"; } +fi + +init_stats() { + NUM_TOTAL=0 # Number of tests that were run + NUM_FAIL=0 # Number of tests that failed unexpectedly + NUM_BROKEN=0 # Number of tests that failed expectedly + NUM_PASS=0 # Number of tests that passed expectedly + NUM_FIXED=0 # Number of tests that passed unexpectedly +} + +get_romcc() { + ROMCC="$BASEDIR/romcc" + if [ ! -f "$ROMCC" ]; then + echo "romcc not found! Please run \"make romcc\"." + exit 1 + fi +} + +init_testing() { + init_stats + get_romcc +} + +show_stats() { + printf "passed: %s\t(%s newly fixed)\n" $NUM_PASS $NUM_FIXED + printf "failed: %s\t(%s known broken)\n" $NUM_FAIL $NUM_BROKEN + printf "total: %s\n" $NUM_TOTAL +} + +is_xfail() { + local t + for t in $XFAIL_TESTS; do + if [ "$t" = "$1" ]; then + return 0 + fi + done + return 1 +} + +pass() { + NUM_TOTAL=$((NUM_TOTAL + 1)) + NUM_PASS=$((NUM_PASS + 1)) + + green "passed" + if is_xfail "$(basename "$1")"; then + blue " (fixed)" + NUM_FIXED=$((NUM_FIXED + 1)) + fi + echo +} + +fail() { + NUM_TOTAL=$((NUM_TOTAL + 1)) + NUM_FAIL=$((NUM_FAIL + 1)) + + red "failed" + if is_xfail "$(basename "$1")"; then + blue " (known broken)" + NUM_BROKEN=$((NUM_BROKEN + 1)) + fi + echo +} + +run_simple_test() { + # TODO: "timeout" is not POSIX compliant. Use something that is. + timeout 60 "$ROMCC" $1 "$2" -o "$BUILDDIR/dummy.S" +} + +run_simple_tests() { + echo "Running simple tests..." + + local t + for t in $(find "$BASEDIR/tests" -name 'simple_test*.c'); do + printf "%s" "$(basename "$t")" + + local result=pass + local logfile="$LOGDIR/$(basename "$t").log" + rm "$logfile" >/dev/null 2>&1 + for opt in "" "-O" "-O2" "-mmmx" "-msse" "-mmmx -msse" \ + "-O -mmmx" "-O -msse" "-O -mmmx -msse" \ + "-O2 -mmmx" "-O2 -msse" "-O2 -mmmx -msse"; do + if run_simple_test "$opt" "$t" \ + >> "$logfile" 2>&1; then + printf . + else + result=fail + break + fi + done + printf " " + $result "$t" + done + + echo +} + +run_linux_test() { + local base="$(basename "$1")" + + # TODO: "timeout" is not POSIX compliant. Use something that is. + + timeout 60 "$ROMCC" "$1" -o "$BUILDDIR/$base.S" || return 1 + as --32 "$BUILDDIR/$base.S" -o "$BUILDDIR/$base.o" || return 1 + ld -m elf_i386 -T "$BASEDIR/tests/ldscript.ld" \ + "$BUILDDIR/$base.o" -o "$BUILDDIR/$base.elf" || return 1 + timeout 60 "$BUILDDIR/$base.elf" > "$BUILDDIR/$base.out" || return 1 + + diff -u "$BASEDIR/results/${base%.c}.out" "$BUILDDIR/$base.out" +} + +run_linux_tests() { + echo "Running linux tests..." + + local t + for t in $(find "$BASEDIR/tests" -name 'linux_test*.c'); do + printf "%s... " "$(basename "$t")" + + local logfile="$LOGDIR/$(basename "$t").log" + if run_linux_test "$t" > "$logfile" 2>&1; then + pass "$t" + else + fail "$t" + fi + done + + echo +} + + +if [ $# -ne 1 ]; then + usage +fi + +CLASS="$1" + +case "$CLASS" in + all) + init_testing + run_simple_tests + run_linux_tests + show_stats + ;; + simple) + init_testing + run_simple_tests + show_stats + ;; + linux) + init_testing + run_linux_tests + show_stats + ;; + *) + echo "Invalid test class $CLASS" + echo + usage + ;; +esac + +if [ $NUM_FAIL -ne $NUM_BROKEN ]; then + exit 1 +fi -- cgit v1.2.3