Created new regression testing framework with a number of testcases (many more

can be added). Also changed 'test' target in makefile to run the new
tester.sh. (Hopefully, we should be able to remove all the tests/*.mk files
soon.)

3 files changed, 409 insertions, 1 deletions

diff --git a/Makefile b/Makefile
index c28e621..c3c0f71 100644
--- a/Makefile
+++ b/Makefile
@@ -317,7 +317,10 @@ loop.h: mk_loop_h.sh
 	@ $(SHELL) $< > $@
 
 test tests:
-	cd tests && $(MAKE) all
+	# old way of doing it
+	#cd tests && $(MAKE) all
+	# new way of doing it
+	cd tests && ./tester.sh
 
 clean:
 	- cd tests && $(MAKE) clean
diff --git a/tests/testcases b/tests/testcases
new file mode 100644
index 0000000..1838fa0
--- /dev/null
+++ b/tests/testcases
@@ -0,0 +1,301 @@
+# testcases
+#
+# This file should be filled with test cases to test applets that:
+#
+#  - can somehow produce output (we can't test sync or sleep)
+#  - have a GNU (or other) counterpart
+#  - are not interactive (don't require a ^C or anything)
+#  - don't require extensive setup or cleanup (a litte setup is fine)
+#  - don't have huge and possibly damaging effects (fsck, swapoff)
+#
+# If possible, a test case should be made that tests each option the applet
+# supports. When a new option is added, a new test case should be written for
+# it. When somebody reports a bug with a testcase, that testcase should be
+# added here as well.
+#
+# Some other guidelines to follow:
+#
+#  - please try to keep applets alphabetized, it will make life easier
+#  - use the file tester.sh or testcases when you need to do a non-destructive
+#    test on a file (i.e. cat, md5sum)
+#  - try to make the applet you're testing the first thing on the line (this
+#    not always possible)
+#  - (???) if you have to create a temporary file, call it TMPFILE
+
+
+# ar
+
+# basename
+basename `pwd`
+
+# cat
+cat tester.sh
+
+# chmod
+# chown
+# chgrp
+# chroot
+# chvt - can't be tested here
+# clear - can't be tested here
+# cmp
+# cp
+# mv
+
+# cut
+echo "1234" | cut -c1
+echo "f1	f2" | cut -f2
+
+# date
+date
+date -R
+date -u
+date +%d/%m/%y
+
+# dc - needs an input file
+# dd
+# deallocvt
+
+# df
+df
+df .
+df -k
+df -h
+df -m
+
+# dirname
+dirname `pwd`
+
+# dmesg (XXX: change the silly cmd business in the source)
+dmesg
+dmesg -n 8
+dmesg -s 512
+# I really don't want to do this next one
+#dmesg -c
+
+# dos2unix - needs an input file
+# dpkg
+# dpkg_deb
+
+# du
+du
+du -s
+du -l
+du -k
+du -h
+du -m
+
+# dumpkmap - no counterprt?
+# dutmp - no counterprt?
+
+# echo
+echo "foo bar baz"
+echo -n "no newline"
+
+
+# expr
+expr 1 \| 1
+expr 1 \| 0
+expr 0 \| 1
+expr 0 \| 0
+
+expr 1 \& 1
+expr 1 \& 0
+expr 0 \& 1
+expr 0 \& 0
+
+expr 0 \< 1
+expr 1 \< 0
+
+expr 1 \> 0
+expr 0 \> 1
+
+expr 0 \<= 1
+expr 1 \<= 0
+expr 1 \<= 1
+
+expr 1 \>= 0
+expr 0 \>= 1
+expr 1 \>= 1
+
+expr 1 + 2
+expr 2 - 1
+expr 2 \* 3
+expr 12 / 2
+expr 12 % 5
+
+# somebody else can do all the string stuff
+
+
+# fbset - can't be tested here
+# fdflush
+# find
+find .
+
+# free
+free
+
+# freeramdisk
+# fsck.minix - won't test
+# getopt
+# grep
+# gunzip
+# gzip
+# halt
+
+# head
+head tester.sh
+head -n 2 tester.sh
+
+# hostid
+hostid
+
+# hostname
+hostname
+hostname -s
+hostname -i
+hostname -d
+# not going to do this next one
+#hostname -F
+
+# id
+id
+id -u
+id -g
+id -ur
+id -un
+
+
+# ifconfig
+#ifconfig
+#ifconfig eth0
+#ifconfig lo
+
+# init - won't test
+# insmod - won't test
+
+# kill
+#kill -l
+# not going to do any more
+
+# length
+# ln
+# loadacm
+# loadfont
+# loadkmap
+# logger
+# logname
+# ls
+ls
+ls -l
+ls -s
+ls -h
+ls -1
+
+# lsmod
+lsmod
+
+# makedevs
+
+# md5sum
+md5sum tester.sh
+
+# mkdir
+# mkfifo
+# mkfs.minix - won't test
+# mknod
+# mkswap - won't test
+# mktemp
+# more - can't test: interactive
+
+# mount
+mount
+# not going to test any more
+
+# mt
+# nc
+# nfsmount
+# nslookup
+# ping - can't test: interactive (needs ^C)
+# pivot_root
+# poweroff - won't test
+# printf
+# ps - there's lotsa differences between busybox ps and any other ps
+
+# pwd
+pwd
+
+# rdate - won't test
+
+# readlink
+ln -sf tester.sh LINKFILE
+readlink LINKFILE
+rm -f LINKFILE
+
+# reboot - won't test
+# renice - won't test
+# reset - can't test: no output
+# rm
+# rmdir
+# rmmod - won't test: dangerous
+# route
+# rpmunpack
+# sed - we can do some one-liners here; probably needs it's own input file
+# setkeycodes
+# sh - this should probably have it's own testcase or input file
+# sleep - can't test: produces no output
+# sort
+# stty
+# swapon - won't test: dangerous
+# swapoff - won't test: dangerous
+# sync - can't test: no output
+# syslogd
+
+# tail
+tail tester.sh
+tail -n 2  tester.sh
+
+# tar
+# tee
+# telnet
+# test
+# tftp
+# touch
+# tr
+true ; echo $?
+false ; echo $?
+# tty
+# umount
+# uname
+# uniq
+# unix2dos
+# update
+
+uptime
+
+# usleep
+# uudecode
+# uuencode
+# watchdog
+
+# wc
+wc tester.sh
+wc -c tester.sh
+wc -w tester.sh
+wc -l tester.sh
+wc -L tester.sh
+
+# wget
+
+# which
+which ls
+
+# whoami
+whoami
+
+# xargs
+# (for some reason, the > redirection isn't working)
+#ls -1 > TMPFILE
+#xargs md5sum < TMPFILE
+#rm -f TMPFILE
+
+# yes - can't test: interactive (needs ^C)
diff --git a/tests/tester.sh b/tests/tester.sh
new file mode 100755
index 0000000..bb49609
--- /dev/null
+++ b/tests/tester.sh
@@ -0,0 +1,104 @@
+#!/bin/bash
+#
+# tester.sh - reads testcases from file and tests busybox applets vs GNU
+# counterparts
+#
+
+# set up defaults (can be changed with cmd-line options)
+BUSYBOX=../busybox
+TESTCASES=testcases
+LOGFILE=tester.log
+BB_OUT=bb.out
+GNU_OUT=gnu.out
+SETUP=""
+CLEANUP=""
+
+# internal-use vars
+fail_only=0
+
+
+while getopts 'p:t:l:b:g:s:c:f' opt
+do
+	case $opt in
+		p) BUSYBOX=$OPTARG; ;;
+		t) TESTCASES=$OPTARG; ;;
+		l) LOGFILE=$OPTARG; ;;
+		b) BB_OUT=$OPTARG; ;;
+		g) GNU_OUT=$OPTARG; ;;
+		s) SETUP=$OPTARG; ;;
+		c) CLEANUP=$OPTARG; ;;
+		f) fail_only=1; ;;
+		*)
+			echo "usage: $0 [-ptlbgsc]"
+			echo "  -p PATH  path to busybox executable"
+			echo "  -t FILE  run testcases in FILE"
+			echo "  -l FILE  log test results in FILE"
+			echo "  -b FILE  store temporary busybox output in FILE"
+			echo "  -g FILE  store temporary GNU output in FILE"
+			echo "  -s FILE  (setup) run commands in FILE before testcases"
+			echo "  -c FILE  (cleanup) run commands in FILE after testcases"
+			echo "  -f       display only testcases that fail"
+			exit 1
+			;;
+	esac
+done
+#shift `expr $OPTIND - 1`
+
+
+# do normal setup
+[ -e $LOGFILE ] && rm $LOGFILE
+unalias -a	# gets rid of aliases that might create different output
+
+# do extra setup (if any)
+if [ ! -z $SETUP ] 
+then
+	echo "running setup commands in $SETUP"
+	sh $SETUP
+	# XXX: Would 'eval' or 'source' work better instead of 'sh'?
+fi
+
+
+# go through each line in the testcase file
+cat $TESTCASES | while read line
+do
+	#echo $line
+	# only process non-blank lines and non-comment lines
+	if [ "$line" ]
+	then
+		if [ `echo "$line" | cut -c1` != "#" ]
+		then
+			[ $fail_only -eq 0 ] && echo "testing: $line" | tee -a $LOGFILE
+
+			# test if the applet was compiled into busybox
+			applet=`echo $line | cut -d' ' -f1`
+			$BUSYBOX 2>&1 | grep -qw $applet
+			if [ $? -eq 1 ]
+			then
+				echo "WHOOPS: $applet not compiled into busybox" | tee -a $LOGFILE
+			else
+				$BUSYBOX $line > $BB_OUT
+				$line > $GNU_OUT
+				diff -q $BB_OUT $GNU_OUT > /dev/null
+				if [ $? -eq 1 ]
+				then
+					echo "FAILED: $line" | tee -a $LOGFILE
+					diff -u $BB_OUT $GNU_OUT >> $LOGFILE 
+				fi
+			fi
+		fi
+	fi
+done
+
+echo "Finished. Results are in $LOGFILE"
+
+
+# do normal cleanup
+rm -f $BB_OUT $GNU_OUT
+
+# do extra cleanup (if any)
+if [ ! -z $CLEANUP ] 
+then
+	echo "running cleanup commands in $CLEANUP"
+	sh $CLEANUP
+	# XXX: Would 'eval' or 'source' work better instead of 'sh'?
+fi