summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTodd Lyons <tlyons@exim.org>2012-07-09 08:38:58 -0700
committerTodd Lyons <tlyons@exim.org>2012-07-09 10:24:55 -0700
commitc1c469dbe99c0521df4dff0eb057622416886fae (patch)
treecefb84b8a68e252278ca9eb1eed474d36fc9f958
parent6754ce8c78c72e82a2da5a5987bd174c80edb532 (diff)
Add CONTINUE to runtest script
-rw-r--r--test/README5
-rwxr-xr-xtest/runtest45
2 files changed, 46 insertions, 4 deletions
diff --git a/test/README b/test/README
index ef91049cc..7e778eee7 100644
--- a/test/README
+++ b/test/README
@@ -237,6 +237,11 @@ is as follows:
There are some options for the ./runtest script itself:
+ -CONTINUE This will allow the script to move past some failing tests. It will
+ write a simple failure line with the test number in a temporary
+ logfile test/failed-summary.log. Unexpected exit codes will still
+ stall the test execution and require interaction.
+
-DEBUG This option is for debugging the test script. It causes some
tracing information to be output.
diff --git a/test/runtest b/test/runtest
index 2756348f9..0514c05f5 100755
--- a/test/runtest
+++ b/test/runtest
@@ -36,7 +36,9 @@ $gnutls_dh_bits_normal = 2236;
$cf = "bin/cf -exact";
$cr = "\r";
$debug = 0;
+$force_continue = 0;
$force_update = 0;
+$log_failed_filename = "failed-summary.log";
$more = "less -XF";
$optargs = "";
$save_output = 0;
@@ -1006,16 +1008,43 @@ return $yield;
# Arguments: [0] the prompt string
# [1] if there is a U in the prompt and $force_update is true
+# [2] if there is a C in the prompt and $force_continue is true
# Returns: nothing (it sets $_)
sub interact{
print $_[0];
if ($_[1]) { $_ = "u"; print "... update forced\n"; }
+ elsif ($_[2]) { $_ = "c"; print "... continue forced\n"; }
else { $_ = <T>; }
}
+##################################################
+# Subroutine to log in force_continue mode #
+##################################################
+
+# In force_continue mode, we just want a terse output to a statically
+# named logfile. If multiple files in same batch (stdout, stderr, etc)
+# all have mismatches, it will log multiple times.
+#
+# Arguments: [0] the logfile to append to
+# [1] the testno that failed
+# Returns: nothing
+
+
+
+sub log_failure {
+ my $logfile = shift();
+ my $testno = shift();
+ my $detail = shift() || '';
+ if ( open(my $fh, ">>", $logfile) ) {
+ print $fh "Test $testno $detail failed\n";
+ close $fh;
+ }
+}
+
+
##################################################
# Subroutine to compare one output file #
@@ -1056,6 +1085,7 @@ if (! -e $sf)
print "Continue, Show, or Quit? [Q] ";
$_ = <T>;
tests_exit(1) if /^q?$/i;
+ log_failure($log_failed_filename, $testno, $rf) if (/^c$/i && $force_continue);
return 0 if /^c$/i;
last if (/^s$/);
}
@@ -1074,8 +1104,9 @@ if (! -e $sf)
print "\n";
for (;;)
{
- interact("Continue, Update & retry, Quit? [Q] ", $force_update);
+ interact("Continue, Update & retry, Quit? [Q] ", $force_update, $force_continue);
tests_exit(1) if /^q?$/i;
+ log_failure($log_failed_filename, $testno, $rsf) if (/^c$/i && $force_continue);
return 0 if /^c$/i;
last if (/^u$/i);
}
@@ -1191,8 +1222,9 @@ if (-e $sf)
print "\n";
for (;;)
{
- interact("Continue, Retry, Update & retry, Quit? [Q] ", $force_update);
+ interact("Continue, Retry, Update & retry, Quit? [Q] ", $force_update, $force_continue);
tests_exit(1) if /^q?$/i;
+ log_failure($log_failed_filename, $testno, $sf) if (/^c$/i && $force_continue);
return 0 if /^c$/i;
return 1 if /^r$/i;
last if (/^u$/i);
@@ -1342,8 +1374,9 @@ if (! $message_skip)
for (;;)
{
- interact("Continue, Update & retry, or Quit? [Q] ", $force_update);
+ interact("Continue, Update & retry, or Quit? [Q] ", $force_update, $force_continue);
tests_exit(1) if /^q?$/i;
+ log_failure($log_failed_filename, $testno, "missing email") if (/^c$/i && $force_continue);
last if /^c$/i;
# For update, we not only have to unlink the file, but we must also
@@ -1425,8 +1458,9 @@ if (! $msglog_skip)
for (;;)
{
- interact("Continue, Update, or Quit? [Q] ", $force_update);
+ interact("Continue, Update, or Quit? [Q] ", $force_update, $force_continue);
tests_exit(1) if /^q?$/i;
+ log_failure($log_failed_filename, $testno, "missing msglog") if (/^c$/i && $force_continue);
last if /^c$/i;
if (/^u$/i)
{
@@ -2131,6 +2165,7 @@ while (@ARGV > 0 && $ARGV[0] =~ /^-/)
{
if ($arg eq "-DEBUG") { $debug = 1; $cr = "\n"; next; }
if ($arg eq "-DIFF") { $cf = "diff -u"; next; }
+ if ($arg eq "-CONTINUE"){$force_continue = 1; next; }
if ($arg eq "-UPDATE") { $force_update = 1; next; }
if ($arg eq "-NOIPV4") { $have_ipv4 = 0; next; }
if ($arg eq "-NOIPV6") { $have_ipv6 = 0; next; }
@@ -3428,6 +3463,7 @@ foreach $test (@test_list)
print "\nshow stdErr, show stdOut, Retry, Continue (without file comparison), or Quit? [Q] ";
$_ = <T>;
tests_exit(1) if /^q?$/i;
+ log_failure($log_failed_filename, $testno, "exit code unexpected") if (/^c$/i && $force_continue);
last if /^[rc]$/i;
if (/^e$/i)
{
@@ -3465,6 +3501,7 @@ foreach $test (@test_list)
print "\nShow server stdout, Retry, Continue, or Quit? [Q] ";
$_ = <T>;
tests_exit(1) if /^q?$/i;
+ log_failure($log_failed_filename, $testno, "exit code unexpected") if (/^c$/i && $force_continue);
last if /^[rc]$/i;
if (/^s$/i)