Browse Source

Parallelize error tests and unparallelize all tests

Rather, I unparallelize tests/all.sh if -n is passed and add targets for
it in the Makefile template.

Signed-off-by: Gavin Howard <gavin@yzena.com>
line_libs
Gavin Howard 11 months ago
parent
commit
289ad4a089
Signed by: gavin
GPG Key ID: C08038BDF280D33E
  1. 18
      Makefile.in
  2. 96
      configure.sh
  3. 8
      scripts/functions.sh
  4. 105
      tests/all.sh
  5. 99
      tests/error.sh
  6. 25
      tests/errors.sh
  7. 52
      tests/scripts.sh

18
Makefile.in

@ -280,17 +280,25 @@ help:
@printf ' valgrind_dc runs the dc test suite, if dc has been built,\n'
@printf ' through valgrind\n'
run_all_tests: bc_all_tests timeconst_all_tests dc_all_tests history_all_tests
run_all_tests: bc_all_tests timeconst_all_tests dc_all_tests
run_all_tests_np: bc_all_tests_np timeconst_all_tests dc_all_tests_np
bc_all_tests:
%%BC_ALL_TESTS%%
bc_all_tests_np:
%%BC_ALL_TESTS_NP%%
timeconst_all_tests:
%%TIMECONST_ALL_TESTS%%
dc_all_tests:
%%DC_ALL_TESTS%%
dc_all_tests_np:
%%DC_ALL_TESTS_NP%%
history_all_tests:
%%HISTORY_TESTS%%
@ -311,7 +319,9 @@ test_bc_stdin:
test_bc_read:
@sh tests/read.sh bc %%BC_TEST_EXEC%%
test_bc_errors:
test_bc_errors: test_bc_error_lines%%BC_ERROR_TESTS%%
test_bc_error_lines:
@sh tests/errors.sh bc %%BC_TEST_EXEC%%
test_bc_other:
@ -333,7 +343,9 @@ test_dc_stdin:
test_dc_read:
@sh tests/read.sh dc %%DC_TEST_EXEC%%
test_dc_errors:
test_dc_errors: test_dc_error_lines%%DC_ERROR_TESTS%%
test_dc_error_lines:
@sh tests/errors.sh dc %%DC_TEST_EXEC%%
test_dc_other:

96
configure.sh

@ -446,35 +446,35 @@ gen_file_list() {
# @param extra_math An integer that, if non-zero, activates extra math tests.
# @param time_tests An integer that, if non-zero, tells the test suite to time
# the execution of each test.
gen_tests() {
gen_std_tests() {
_gen_tests_name="$1"
_gen_std_tests_name="$1"
shift
_gen_tests_extra_math="$1"
_gen_std_tests_extra_math="$1"
shift
_gen_tests_time_tests="$1"
_gen_std_tests_time_tests="$1"
shift
_gen_tests_extra_required=$(cat "$scriptdir/tests/extra_required.txt")
_gen_std_tests_extra_required=$(cat "$scriptdir/tests/extra_required.txt")
for _gen_tests_t in $(cat "$scriptdir/tests/$_gen_tests_name/all.txt"); do
for _gen_std_tests_t in $(cat "$scriptdir/tests/$_gen_std_tests_name/all.txt"); do
if [ "$_gen_tests_extra_math" -eq 0 ]; then
if [ "$_gen_std_tests_extra_math" -eq 0 ]; then
if [ -z "${_gen_tests_extra_required##*$_gen_tests_t*}" ]; then
if [ -z "${_gen_std_tests_extra_required##*$_gen_std_tests_t*}" ]; then
printf 'test_%s_%s:\n\t@printf "Skipping %s %s\\n"\n\n' \
"$_gen_tests_name" "$_gen_tests_t" "$_gen_tests_name" \
"$_gen_tests_t" >> "$scriptdir/Makefile"
"$_gen_std_tests_name" "$_gen_std_tests_t" "$_gen_std_tests_name" \
"$_gen_std_tests_t" >> "$scriptdir/Makefile"
continue
fi
fi
printf 'test_%s_%s:\n\t@sh tests/test.sh %s %s %s %s %s\n\n' \
"$_gen_tests_name" "$_gen_tests_t" "$_gen_tests_name" \
"$_gen_tests_t" "$generate_tests" "$time_tests" \
"$_gen_std_tests_name" "$_gen_std_tests_t" "$_gen_std_tests_name" \
"$_gen_std_tests_t" "$generate_tests" "$time_tests" \
"$*" >> "$scriptdir/Makefile"
done
@ -484,15 +484,55 @@ gen_tests() {
# targets.
#
# @param name The name of the calculator to generate test targets for.
gen_test_targets() {
gen_std_test_targets() {
_gen_test_targets_name="$1"
_gen_std_test_targets_name="$1"
shift
_gen_test_targets_tests=$(cat "$scriptdir/tests/${_gen_test_targets_name}/all.txt")
_gen_std_test_targets_tests=$(cat "$scriptdir/tests/${_gen_std_test_targets_name}/all.txt")
for _gen_test_targets_t in $_gen_test_targets_tests; do
printf ' test_%s_%s' "$_gen_test_targets_name" "$_gen_test_targets_t"
for _gen_std_test_targets_t in $_gen_std_test_targets_tests; do
printf ' test_%s_%s' "$_gen_std_test_targets_name" "$_gen_std_test_targets_t"
done
printf '\n'
}
# Generates the proper test targets for each error test to have its own target.
# This allows `make test_bc_errors` and `make test_dc_errors` to run in
# parallel.
#
# @param name Which calculator to generate tests for.
gen_err_tests() {
_gen_err_tests_name="$1"
shift
_gen_err_tests_fs=$(ls "$scriptdir/tests/$_gen_err_tests_name/errors/")
for _gen_err_tests_t in $_gen_err_tests_fs; do
printf 'test_%s_error_%s:\n\t@sh tests/error.sh %s %s %s\n\n' \
"$_gen_err_tests_name" "$_gen_err_tests_t" "$_gen_err_tests_name" \
"$_gen_err_tests_t" "$*" >> "$scriptdir/Makefile"
done
}
# Generates a list of error test targets that will be used as prerequisites for
# other targets.
#
# @param name The name of the calculator to generate test targets for.
gen_err_test_targets() {
_gen_err_test_targets_name="$1"
shift
_gen_err_test_targets_tests=$(ls "$scriptdir/tests/$_gen_err_test_targets_name/errors/")
for _gen_err_test_targets_t in $_gen_err_test_targets_tests; do
printf ' test_%s_error_%s' "$_gen_err_test_targets_name" "$_gen_err_test_targets_t"
done
printf '\n'
@ -907,7 +947,9 @@ executable="BC_EXEC"
tests="test_bc timeconst test_dc"
bc_test="@tests/all.sh bc $extra_math 1 $generate_tests $time_tests \$(BC_EXEC)"
bc_test_np="@tests/all.sh -n bc $extra_math 1 $generate_tests $time_tests \$(BC_EXEC)"
dc_test="@tests/all.sh dc $extra_math 1 $generate_tests $time_tests \$(DC_EXEC)"
dc_test_np="@tests/all.sh -n dc $extra_math 1 $generate_tests $time_tests \$(DC_EXEC)"
timeconst="@tests/bc/timeconst.sh tests/bc/scripts/timeconst.bc \$(BC_EXEC)"
@ -967,6 +1009,7 @@ elif [ "$bc_only" -eq 1 ]; then
executables="bc"
dc_test="@printf 'No dc tests to run\\\\n'"
dc_test_np="@printf 'No dc tests to run\\\\n'"
test_dc_history_prereqs=" test_dc_history_skip"
install_prereqs=" install_execs"
@ -992,6 +1035,7 @@ elif [ "$dc_only" -eq 1 ]; then
executable="DC_EXEC"
bc_test="@printf 'No bc tests to run\\\\n'"
bc_test_np="@printf 'No bc tests to run\\\\n'"
test_bc_history_prereqs=" test_bc_history_skip"
timeconst="@printf 'timeconst cannot be run because bc is not built\\\\n'"
@ -1392,10 +1436,12 @@ if [ "$dc_default_prompt" = "" ]; then
fi
# Generate the test targets and prerequisites.
bc_tests=$(gen_test_targets bc)
bc_tests=$(gen_std_test_targets bc)
bc_script_tests=$(gen_script_test_targets bc)
dc_tests=$(gen_test_targets dc)
bc_err_tests=$(gen_err_test_targets bc)
dc_tests=$(gen_std_test_targets dc)
dc_script_tests=$(gen_script_test_targets dc)
dc_err_tests=$(gen_err_test_targets dc)
# Print out the values; this is for debugging.
if [ "$bc" -ne 0 ]; then
@ -1483,14 +1529,18 @@ contents=$(replace "$contents" "BC_ENABLED" "$bc")
contents=$(replace "$contents" "DC_ENABLED" "$dc")
contents=$(replace "$contents" "BC_ALL_TESTS" "$bc_test")
contents=$(replace "$contents" "BC_ALL_TESTS_NP" "$bc_test_np")
contents=$(replace "$contents" "BC_TESTS" "$bc_tests")
contents=$(replace "$contents" "BC_SCRIPT_TESTS" "$bc_script_tests")
contents=$(replace "$contents" "BC_ERROR_TESTS" "$bc_err_tests")
contents=$(replace "$contents" "BC_TEST_EXEC" "$bc_test_exec")
contents=$(replace "$contents" "TIMECONST_ALL_TESTS" "$timeconst")
contents=$(replace "$contents" "DC_ALL_TESTS" "$dc_test")
contents=$(replace "$contents" "DC_ALL_TESTS_NP" "$dc_test_np")
contents=$(replace "$contents" "DC_TESTS" "$dc_tests")
contents=$(replace "$contents" "DC_SCRIPT_TESTS" "$dc_script_tests")
contents=$(replace "$contents" "DC_ERROR_TESTS" "$dc_err_tests")
contents=$(replace "$contents" "DC_TEST_EXEC" "$dc_test_exec")
contents=$(replace "$contents" "BUILD_TYPE" "$manpage_args")
@ -1551,9 +1601,7 @@ contents=$(replace "$contents" "MAIN_EXEC" "$main_exec")
contents=$(replace "$contents" "EXEC" "$executable")
contents=$(replace "$contents" "TESTS" "$tests")
contents=$(replace "$contents" "BC_TEST" "$bc_test")
contents=$(replace "$contents" "BC_HISTORY_TEST_PREREQS" "$test_bc_history_prereqs")
contents=$(replace "$contents" "DC_TEST" "$dc_test")
contents=$(replace "$contents" "DC_HISTORY_TEST_PREREQS" "$test_dc_history_prereqs")
contents=$(replace "$contents" "HISTORY_TESTS" "$history_tests")
@ -1588,13 +1636,15 @@ printf '%s\n%s\n\n' "$contents" "$SRC_TARGETS" > "$scriptdir/Makefile"
# Generate the individual test targets.
if [ "$bc" -ne 0 ]; then
gen_tests bc "$extra_math" "$time_tests" $bc_test_exec
gen_std_tests bc "$extra_math" "$time_tests" $bc_test_exec
gen_script_tests bc "$extra_math" "$generate_tests" "$time_tests" $bc_test_exec
gen_err_tests bc $bc_test_exec
fi
if [ "$dc" -ne 0 ]; then
gen_tests dc "$extra_math" "$time_tests" $dc_test_exec
gen_std_tests dc "$extra_math" "$time_tests" $dc_test_exec
gen_script_tests dc "$extra_math" "$generate_tests" "$time_tests" $dc_test_exec
gen_err_tests dc $dc_test_exec
fi
cd "$scriptdir"

8
scripts/functions.sh

@ -223,11 +223,9 @@ checkerrtest()
die "$_checkerrtest_d" "produced no error message" "$_checkerrtest_name" "$_checkerrtest_error"
fi
# Display the error messages if not directly running exe.
# This allows the script to print valgrind output.
if [ "$_checkerrtest_exebase" != "bc" ] && [ "$_checkerrtest_exebase" != "dc" ]; then
cat "$_checkerrtest_out"
fi
# To display error messages, uncomment this line. This is useful when
# debugging.
#cat "$_checkerrtest_out"
}
# Replace a substring in a string with another. This function is the *real*

105
tests/all.sh

@ -32,12 +32,24 @@ testdir=$(dirname "$script")
. "$testdir/../scripts/functions.sh"
# We need to figure out if we should run stuff in parallel.
pll=1
while getopts "n" opt; do
case "$opt" in
n) pll=0 ; shift ; set -e ;;
?) usage "Invalid option: $opt" ;;
esac
done
# Command-line processing.
if [ "$#" -ge 1 ]; then
d="$1"
shift
else
err_exit "usage: $script dir [run_extra_tests] [run_stack_tests] [gen_tests] [time_tests] [exec args...]" 1
err_exit "usage: $script [-n] dir [run_extra_tests] [run_stack_tests] [gen_tests] [time_tests] [exec args...]" 1
fi
if [ "$#" -lt 1 ]; then
@ -109,47 +121,92 @@ while read t; do
fi
fi
sh "$testdir/test.sh" "$d" "$t" "$generate_tests" "$time_tests" "$exe" "$@" &
pids="$pids $!"
if [ "$pll" -ne 0 ]; then
sh "$testdir/test.sh" "$d" "$t" "$generate_tests" "$time_tests" "$exe" "$@" &
pids="$pids $!"
else
sh "$testdir/test.sh" "$d" "$t" "$generate_tests" "$time_tests" "$exe" "$@"
fi
done < "$testdir/$d/all.txt"
# stdin tests.
sh "$testdir/stdin.sh" "$d" "$exe" "$@" &
pids="$pids $!"
if [ "$pll" -ne 0 ]; then
sh "$testdir/stdin.sh" "$d" "$exe" "$@" &
pids="$pids $!"
else
sh "$testdir/stdin.sh" "$d" "$exe" "$@"
fi
# Script tests.
sh "$testdir/scripts.sh" "$d" "$extra" "$run_stack_tests" "$generate_tests" \
"$time_tests" "$exe" "$@" &
pids="$pids $!"
if [ "$pll" -ne 0 ]; then
sh "$testdir/scripts.sh" "$d" "$extra" "$run_stack_tests" "$generate_tests" \
"$time_tests" "$exe" "$@" &
pids="$pids $!"
else
sh "$testdir/scripts.sh" -n "$d" "$extra" "$run_stack_tests" "$generate_tests" \
"$time_tests" "$exe" "$@"
fi
# Read tests.
sh "$testdir/read.sh" "$d" "$exe" "$@" &
pids="$pids $!"
if [ "$pll" -ne 0 ]; then
sh "$testdir/read.sh" "$d" "$exe" "$@" &
pids="$pids $!"
else
sh "$testdir/read.sh" "$d" "$exe" "$@"
fi
# Error tests.
sh "$testdir/errors.sh" "$d" "$exe" "$@" &
pids="$pids $!"
if [ "$pll" -ne 0 ]; then
sh "$testdir/errors.sh" "$d" "$exe" "$@" &
pids="$pids $!"
else
sh "$testdir/errors.sh" "$d" "$exe" "$@"
fi
# Test all the files in the errors directory. While the other error test (in
# tests/errors.sh) does a test for every line, this does one test per file, but
# it runs the file through stdin and as a file on the command-line.
for testfile in $testdir/$d/errors/*.txt; do
b=$(basename "$testfile")
if [ "$pll" -ne 0 ]; then
sh "$testdir/error.sh" "$d" "$b" "$@" &
pids="$pids $!"
else
sh "$testdir/error.sh" "$d" "$b" "$@"
fi
done
# Other tests.
sh "$testdir/other.sh" "$d" "$extra" "$exe" "$@" &
pids="$pids $!"
if [ "$pll" -ne 0 ]; then
sh "$testdir/other.sh" "$d" "$extra" "$exe" "$@" &
pids="$pids $!"
else
sh "$testdir/other.sh" "$d" "$extra" "$exe" "$@"
fi
exit_err=0
if [ "$pll" -ne 0 ]; then
for p in $pids; do
exit_err=0
wait "$p"
err="$?"
for p in $pids; do
if [ "$err" -ne 0 ]; then
printf 'A test failed!\n'
exit_err=1
wait "$p"
err="$?"
if [ "$err" -ne 0 ]; then
printf 'A test failed!\n'
exit_err=1
fi
done
if [ "$exit_err" -ne 0 ]; then
exit 1
fi
done
if [ "$exit_err" -ne 0 ]; then
exit 1
fi
printf '\nAll %s tests passed.\n' "$d"

99
tests/error.sh

@ -0,0 +1,99 @@
#! /bin/sh
#
# SPDX-License-Identifier: BSD-2-Clause
#
# Copyright (c) 2018-2021 Gavin D. Howard and contributors.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
script="$0"
testdir=$(dirname "$script")
. "$testdir/../scripts/functions.sh"
outputdir=${BC_TEST_OUTPUT_DIR:-$testdir}
# Command-line processing.
if [ "$#" -lt 2 ]; then
printf 'usage: %s dir test [exec args...]\n' "$script"
exit 1
else
d="$1"
shift
t="$1"
shift
fi
if [ "$#" -lt 1 ]; then
exe="$testdir/../bin/$d"
else
exe="$1"
shift
fi
# I use these, so unset them to make the tests work.
unset BC_ENV_ARGS
unset BC_LINE_LENGTH
unset DC_ENV_ARGS
unset DC_LINE_LENGTH
out="$outputdir/${d}_outputs/error_results_${t}"
outdir=$(dirname "$out")
# Make sure the directory exists.
if [ ! -d "$outdir" ]; then
mkdir -p "$outdir"
fi
# Set stuff for the correct calculator.
if [ "$d" = "bc" ]; then
opts="-l"
halt="halt"
read_call="read()"
read_expr="${read_call}\n5+5;"
else
opts="-x"
halt="q"
fi
testfile="$testdir/$d/errors/$t"
printf 'Running %s error file %s...' "$d" "$t"
printf '%s\n' "$halt" | "$exe" "$@" $opts "$testfile" 2> "$out" > /dev/null
err="$?"
checkerrtest "$d" "$err" "$testfile" "$out" "$exebase" > /dev/null
printf 'pass\n'
printf 'Running %s error file %s through cat...' "$d" "$t"
cat "$testfile" | "$exe" "$@" $opts 2> "$out" > /dev/null
err="$?"
checkcrash "$d" "$err" "$testfile"
printf 'pass\n'

25
tests/errors.sh

@ -147,28 +147,3 @@ for testfile in $testdir/$d/*errors.txt; do
printf 'pass\n'
done
# Test all the files in the errors directory. While the loop above does one test
# for every line, this does one test per file, but it runs the file through
# stdin and as a file on the command-line.
for testfile in $testdir/$d/errors/*.txt; do
printf 'Running %s error file %s...' "$d" "$testfile"
printf '%s\n' "$halt" | "$exe" "$@" $opts "$testfile" 2> "$out" > /dev/null
err="$?"
checkerrtest "$d" "$err" "$testfile" "$out" "$exebase"
printf 'pass\n'
printf 'Running %s error file %s through cat...' "$d" "$testfile"
cat "$testfile" | "$exe" "$@" $opts 2> "$out" > /dev/null
err="$?"
checkcrash "$d" "$err" "$testfile"
printf 'pass\n'
done

52
tests/scripts.sh

@ -33,9 +33,21 @@ testdir=$(dirname "${script}")
pids=""
# We need to figure out if we should run stuff in parallel.
pll=1
while getopts "n" opt; do
case "$opt" in
n) pll=0 ; shift ; set -e ;;
?) usage "Invalid option: $opt" ;;
esac
done
# Command-line processing.
if [ "$#" -eq 0 ]; then
printf 'usage: %s dir [run_extra_tests] [run_stack_tests] [generate_tests] [time_tests] [exec args...]\n' "$script"
printf 'usage: %s [-n] dir [run_extra_tests] [run_stack_tests] [generate_tests] [time_tests] [exec args...]\n' "$script"
exit 1
else
d="$1"
@ -85,26 +97,36 @@ scripts=$(cat "$scriptdir/all.txt")
for s in $scripts; do
f=$(basename "$s")
sh "$testdir/script.sh" "$d" "$f" "$run_extra_tests" "$run_stack_tests" \
"$generate" "$time_tests" "$exe" "$@" &
pids="$pids $!"
if [ "$pll" -ne 0 ]; then
sh "$testdir/script.sh" "$d" "$f" "$run_extra_tests" "$run_stack_tests" \
"$generate" "$time_tests" "$exe" "$@" &
pids="$pids $!"
else
sh "$testdir/script.sh" "$d" "$f" "$run_extra_tests" "$run_stack_tests" \
"$generate" "$time_tests" "$exe" "$@"
fi
done
exit_err=0
if [ "$pll" -ne 0 ]; then
for p in $pids; do
exit_err=0
wait "$p"
err="$?"
for p in $pids; do
if [ "$err" -ne 0 ]; then
printf 'A script failed!\n'
exit_err=1
fi
wait "$p"
err="$?"
done
if [ "$err" -ne 0 ]; then
printf 'A script failed!\n'
exit_err=1
fi
done
if [ "$exit_err" -ne 0 ]; then
exit 1
fi
if [ "$exit_err" -ne 0 ]; then
exit 1
fi

Loading…
Cancel
Save