summaryrefslogtreecommitdiff
path: root/tests/tester.sh
blob: 09ba750ece3700284397b8d3eed533317672c9a8 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
#!/bin/bash
#
# tester.sh - reads testcases from file and tests busybox applets vs GNU
# counterparts
#
# This should be run from within the tests/ directory. Before you run it, you
# should compile up a busybox that has all applets and all features turned on.

# set up defaults (can be changed with cmd-line options)
BUSYBOX=../busybox
TESTCASES=testcases
LOGFILE=tester.log
CONFIG_OUT=bb.out
GNU_OUT=gnu.out
SETUP=""
CLEANUP=""
KEEPTMPFILES="no"
DEBUG=2


#while getopts 'p:t:l:b:g:s:c:kd:' opt
while getopts 'p:t:l:s:c:kd:' opt
do
	case $opt in
		p) BUSYBOX=$OPTARG; ;;
		t) TESTCASES=$OPTARG; ;;
		l) LOGFILE=$OPTARG; ;;
#		b) CONFIG_OUT=$OPTARG; ;;
#		g) GNU_OUT=$OPTARG; ;;
		s) SETUP=$OPTARG; ;;
		c) CLEANUP=$OPTARG; ;;
		k) KEEPTMPFILES="yes"; ;;
		d) DEBUG=$OPTARG; ;;
		*)
			echo "usage: $0 [-ptlbgsc]"
			echo "  -p PATH  path to busybox executable (default=$BUSYBOX)"
			echo "  -t FILE  run testcases in FILE (default=$TESTCASES)"
			echo "  -l FILE  log test results in FILE (default=$LOGFILE)"
#			echo "  -b FILE  store temporary busybox output in FILE"
#			echo "  -g FILE  store temporary GNU output in FILE"
			echo "  -s FILE  (setup) run commands in FILE before testcases"
			echo "  -c FILE  (cleanup) run commands in FILE after testcases"
			echo "  -k       keep temporary output files (don't delete them)"
			echo "  -d NUM   set level of debugging output"
			echo "           0 = no output"
			echo "           1 = output failures / whoops lines only"
			echo "           2 = (default) output setup / cleanup msgs and testcase lines"
			echo "           3+= other debug noise (internal stuff)"
			exit 1
			;;
	esac
done
#shift `expr $OPTIND - 1`


# maybe print some debug output
if [ $DEBUG -ge 3 ]
then
	echo "BUSYBOX=$BUSYBOX"
	echo "TESTCASES=$TESTCASES"
	echo "LOGFILE=$LOGFILE"
	echo "CONFIG_OUT=$CONFIG_OUT"
	echo "GNU_OUT=$GNU_OUT"
	echo "SETUP=$SETUP"
	echo "CLEANUP=$CLEANUP"
	echo "DEBUG=$DEBUG"
fi


# do sanity checks
if [ ! -e $BUSYBOX ]
then
	echo "Busybox executable: $BUSYBOX not found!"
	exit 1
fi

if [ ! -e $TESTCASES ]
then
	echo "Testcases file: $TESTCASES not found!"
	exit 1
fi


# do normal setup
[ -e $LOGFILE ] && rm $LOGFILE
unalias -a	# gets rid of aliases that might create different output


# do extra setup (if any)
if [ ! -z "$SETUP" ] 
then
	[ $DEBUG -ge 2 ] && echo "running setup commands in $SETUP"
	source $SETUP
fi


# go through each line in the testcase file
cat $TESTCASES | while read line
do
	#echo $line
	# only process non-blank lines and non-comment lines
	if [ "$line" ]
	then
		if [ `echo "$line" | cut -c1` != "#" ]
		then

			# test if the applet was compiled into busybox
			# (this only tests the applet at the beginning of the line)
			#applet=`echo $line | cut -d' ' -f1`
			applet=`echo $line | sed 's/\(^[^ ;]*\)[ ;].*/\1/'`
			$BUSYBOX 2>&1 | grep -qw $applet
			if [ $? -eq 1 ]
			then
				echo "WHOOPS: $applet not compiled into busybox" | tee -a $LOGFILE
			else

				# execute line using gnu / system programs
				[ $DEBUG -ge 2 ] && echo "testing: $line" | tee -a $LOGFILE
				sh -c "$line" > $GNU_OUT

				# change line to include "busybox" before every statement
				line="$BUSYBOX $line"
				# is this a bash-2-ism?
				# line=${line//;/; $BUSYBOX }
				# line=${line//|/| $BUSYBOX }
				# assume $BUSYBOX has no commas
				line=`echo "$line" | sed -e 's,;,; '$BUSYBOX, \
				                       -e 's, |, | '$BUSYBOX,`

				# execute line using busybox programs
				[ $DEBUG -ge 2 ] && echo "testing: $line" | tee -a $LOGFILE
				sh -c "$line" > $CONFIG_OUT

				# see if they match
				diff -q $CONFIG_OUT $GNU_OUT > /dev/null
				if [ $? -eq 1 ]
				then
					[ $DEBUG -ge 1 ] && echo "FAILED: $line" | tee -a $LOGFILE
					diff -u $CONFIG_OUT $GNU_OUT >> $LOGFILE 
				fi
			fi
		fi
	fi
done

[ $DEBUG -gt 0 ] && echo "Finished. Results are in $LOGFILE"


# do normal cleanup
[ "$KEEPTMPFILES" = "no" ] && rm -f $CONFIG_OUT $GNU_OUT


# do extra cleanup (if any)
if [ ! -z "$CLEANUP" ] 
then
	[ $DEBUG -ge 2 ] && echo "running cleanup commands in $CLEANUP"
	source $CLEANUP
fi