Testsuite: support "anything but" returncode script lines

5 files changed, 47 insertions, 24 deletions

diff --git a/test/README b/test/README
index d013f7823..c0bfa04f1 100644
--- a/test/README
+++ b/test/README
@@ -681,7 +681,12 @@ script. For example:
   # -bS Use of HELO/RSET
 
 A line consisting just of digits is interpreted as the expected return code
-for the command that follows. The default expectation when no such line exists
+for the command that follows.
+
+A line consisting of a tilde (~) followed by digits indicates a non-expected
+return code for the command that follows.
+
+The default expectation when neither such line exists
 is a zero return code. For example, here is a complete test script, containing
 just one command:
 
diff --git a/test/runtest b/test/runtest
index 204e09e76..629b45dbf 100755
--- a/test/runtest
+++ b/test/runtest
@@ -2305,6 +2305,7 @@ system($cmd);
 # Arguments: the current test number
 #            reference to the subtest number, holding previous value
 #            reference to the expected return code value
+#            reference to flag for not-expected return value
 #            reference to where to put the command name (for messages)
 #            auxiliary information returned from a previous run
 #
@@ -2320,17 +2321,18 @@ system($cmd);
 sub run_command{
 my($testno) = $_[0];
 my($subtestref) = $_[1];
-my($commandnameref) = $_[3];
-my($aux_info) = $_[4];
+my($commandnameref) = $_[4];
+my($aux_info) = $_[5];
 my($yield) = 1;
 
 our %ENV = map { $_ => $ENV{$_} } grep { /^(?:USER|SHELL|PATH|TERM|EXIM_TEST_.*)$/ } keys %ENV;
 
-if (/^(\d+)\s*(?:([A-Z]+)=(\S+))?$/)                # Handle unusual return code
+if (/^(~)?(\d+)\s*(?:([A-Z]+)=(\S+))?$/)                # Handle unusual return code
   {
-  my($r) = $_[2];
-  $$r = $1 << 8;
-  $ENV{$2} = $3 if (defined $2);
+  my($r, $rn) = ($_[2], $_[3]);
+  $$r = $2 << 8;
+  $$rn = 1 if (defined $1);
+  $ENV{$3} = $4 if (defined $3);
   $_ = <SCRIPT>;
   return 4 if !defined $_;       # Missing command
   $lineno++;
@@ -4195,6 +4197,7 @@ DIR: for (my $i = 0; $i < @test_dirs; $i++)
   # range that was selected.
 
   @testlist = grep { $_ ~~ @wanted } grep { /^\d+(?:\.\d+)?$/ } map { basename $_ } glob "scripts/$testdir/*";
+
   tests_exit(-1, "Failed to read test scripts from `scripts/$testdir/*': $!")
     if not @testlist;
 
@@ -4580,8 +4583,8 @@ foreach $test (@test_list)
     # was run and not waited for (usually a daemon or server startup).
 
     my($commandname) = '';
-    my($expectrc) = 0;
-    my($rc, $run_extra) = run_command($testno, \$subtestno, \$expectrc, \$commandname, $TEST_STATE);
+    my($expectrc, $expect_not) = (0, 0);
+    my($rc, $run_extra) = run_command($testno, \$subtestno, \$expectrc, \$expect_not, \$commandname, $TEST_STATE);
     my($cmdrc) = $?;
 
     if ($debug) {
@@ -4619,12 +4622,15 @@ foreach $test (@test_list)
     # We ran and waited for a command. Check for the expected result unless
     # it died.
 
-    if ($cmdrc != $expectrc && !$sigpipehappened)
+    if (!$sigpipehappened && ($expect_not ? ($cmdrc == $expectrc) : ($cmdrc != $expectrc)))
       {
       printf("** Command $commandno (\"$commandname\", starting at line $subtest_startline)\n");
       if (($cmdrc & 0xff) == 0)
         {
-        printf("** Return code %d (expected %d)", $cmdrc/256, $expectrc/256);
+	if ($expect_not)
+	  { printf("** Return code %d (expected anything but that)", $cmdrc/256); }
+	else
+	  { printf("** Return code %d (expected %d)", $cmdrc/256, $expectrc/256); }
         }
       elsif (($cmdrc & 0xff00) == 0)
         { printf("** Killed by signal %d", $cmdrc & 255); }
diff --git a/test/scripts/0000-Basic/0700 b/test/scripts/0000-Basic/0700
index 7bc8671df..74d19d7b3 100644
--- a/test/scripts/0000-Basic/0700
+++ b/test/scripts/0000-Basic/0700
@@ -46,11 +46,11 @@ exim_msgdate --base36 000000-005XGK-00
 0 TZ=GMT
 exim_msgdate --base36 ZZZZZZ-005XGK-RR
 ****
-255 TZ=GB
+~0 TZ=GB
 exim_msgdate --base36 zzzzzz-005xgk-zz
 ****
 ### An invalid base option
-255 TZ=GMT
+~0 TZ=GMT
 exim_msgdate --base 32 ZZZZZZ
 ****
 ### Message IDs generated on a standard system
@@ -127,19 +127,19 @@ exim_msgdate --localhost_number 9 -base 62 EZZZZZZ-003FPJ-WF
 0 TZ=GMT
 exim_msgdate --localhost_number 9 -base 36 EZZZZZZ-003FPJ-Q0
 ****
-# hostno too big
-255 TZ=GMT
+### hostno too big
+~0 TZ=GMT
 exim_msgdate --localhost_number 11 -base 36 EZZZZZZ-003FPJ-WF
 ****
 0 TZ=GMT
 exim_msgdate -localhost_number 9 --local -base 62 EZZZZZZ-003FPJ-WF
 ****
-# hostno negative
-255 TZ=GMT
+### hostno negative
+~0 TZ=GMT
 exim_msgdate --localhost_number -1 -base 36 EZZZZZZ-003FPJ-WF
 ****
-# hostno too big
-255 TZ=GMT
+### hostno too big
+~0 TZ=GMT
 exim_msgdate --localhost_number 19 -base 62 EZZZZZZ-003FPJ-WF
 ****
 #
@@ -187,16 +187,16 @@ exim_msgdate -l -u -z -localhost_number=13 000000 1PANS3 zzzzzz
 exim_msgdate -l -u -z -localhost_number=13 000000 1PANS3 ZZZZZZ
 ****
 ### All msg-id types, all zones,  localhost_number = 20
-255 TZ=GB
+~0 TZ=GB
 exim_msgdate -l -u -z -localhost_number=20 E000000-005XGK-00
 ****
-255 TZ=GMT
+~0 TZ=GMT
 exim_msgdate -l -u -z -localhost_number=20 000000-005XGK-00 ZZZZZZ-005XGK-zz 1PANS3-003FPJ-TW
 ****
-255 TZ=GB
+~0 TZ=GB
 exim_msgdate -l -u -z -localhost_number=20 000000 1PANS3 zzzzzz
 ****
-255 TZ=GMT
+~0 TZ=GMT
 exim_msgdate -l -u -z -localhost_number=20 000000 1PANS3 ZZZZZZ
 ****
 ### All msg-id types, all zones,  base=36
@@ -206,7 +206,7 @@ exim_msgdate -l -u -z --b36 E000000-005XGK-00
 0 TZ=GMT
 exim_msgdate -l -u -z --b36 000000-005XGK-00 ZZZZZZ-005XGK-R0 1PANS3-003FPJ-RR
 ****
-255 TZ=GB
+~0 TZ=GB
 exim_msgdate -l -u -z --b36 000000 1PANS3 zzzzzz
 ****
 0 TZ=GMT
diff --git a/test/stderr/0700 b/test/stderr/0700
index 48aafe811..ff03d69cd 100644
--- a/test/stderr/0700
+++ b/test/stderr/0700
@@ -23,8 +23,11 @@ zzzzzzz not parsed
 ### Print date as seconds since the UNIX epoch.
 ### Show the process id too
 ### Override the value of localhost_number set in the exim configuation file
+### hostno too big
 localhost_number > 10
+### hostno negative
 localhost_number must be a number >=0
+### hostno too big
 localhost_number > 16
 ### From here as 701 - 703
 ### Each msg-id type, all zone
@@ -68,6 +71,9 @@ zzzzzz is not base 36
 ### Print date as seconds since the UNIX epoch.
 ### Show the process id too
 ### Override the value of localhost_number set in the exim configuation file
+### hostno too big
+### hostno negative
+### hostno too big
 ### From here as 701 - 703
 ### Each msg-id type, all zone
 ### All msg-id types, all zones,  localhost_number = 3
diff --git a/test/stdout/0700 b/test/stdout/0700
index 3a7373b67..7769084d9 100644
--- a/test/stdout/0700
+++ b/test/stdout/0700
@@ -51,7 +51,10 @@
 3002-09-30 12:51:45.995000 GMT
 3002-09-30 12:51:45.995000 GMT
 2038-12-24 05:45:35.360000 GMT
+### hostno too big
 3002-09-30 12:51:45.995000 GMT
+### hostno negative
+### hostno too big
 ### From here as 701 - 703
 ### Each msg-id type, all zone
 0.000000	pid 1319504
@@ -228,6 +231,9 @@
 ### Print date as seconds since the UNIX epoch.
 ### Show the process id too
 ### Override the value of localhost_number set in the exim configuation file
+### hostno too big
+### hostno negative
+### hostno too big
 ### From here as 701 - 703
 ### Each msg-id type, all zone
 ### All msg-id types, all zones,  localhost_number = 3