(1) Update version number to 4.64. (2) Ignore callout cached information

that MAIL FROM:<> is rejected when verifying a recipient using a
non-null MAIL address.

12 files changed, 219 insertions, 32 deletions

diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog
index 3187570b8..6e70b4f13 100644
--- a/doc/doc-txt/ChangeLog
+++ b/doc/doc-txt/ChangeLog
@@ -1,4 +1,4 @@
-$Cambridge: exim/doc/doc-txt/ChangeLog,v 1.386 2006/08/21 11:31:43 fanf2 Exp $
+$Cambridge: exim/doc/doc-txt/ChangeLog,v 1.387 2006/09/05 13:24:10 ph10 Exp $
 
 Change log file for Exim from version 4.21
 -------------------------------------------
@@ -11,6 +11,18 @@ TF/01 In the add_headers option to the mail command in an Exim filter,
       header after the first one which had an odd number of characters
       in the field name.
 
+PH/01 If a server that rejects MAIL FROM:<> was the target of a sender
+      callout verification, Exim cached a "reject" for the entire domain. This
+      is correct for most verifications, but it is not correct for a recipient
+      verification with use_sender or use_postmaster set, because in that case
+      the callout does not use MAIL FROM:<>. Exim now distinguishes the special
+      case of MAIL FROM:<> rejection from other early rejections (e.g.
+      rejection of HELO). When verifying a recipient using a non-null MAIL
+      address, the cache is ignored if it shows MAIL FROM:<> rejection.
+      Whatever the result of the callout, the value of the domain cache is
+      left unchanged (for any other kind of callout, getting as far as trying
+      RCPT means that the domain itself is ok).
+
 
 Exim version 4.63
 -----------------
diff --git a/src/src/dbstuff.h b/src/src/dbstuff.h
index 06facfb09..2b51b0ba8 100644
--- a/src/src/dbstuff.h
+++ b/src/src/dbstuff.h
@@ -1,4 +1,4 @@
-/* $Cambridge: exim/src/src/dbstuff.h,v 1.4 2006/02/07 11:19:00 ph10 Exp $ */
+/* $Cambridge: exim/src/src/dbstuff.h,v 1.5 2006/09/05 13:24:10 ph10 Exp $ */
 
 /*************************************************
 *     Exim - an Internet mail transport agent    *
@@ -564,9 +564,11 @@ performed on them. There are two groups of records:
 2. keyed by domain -
      Domain response upto MAIL FROM:<>, postmaster, random local part;
 
-If a record exists, the result field is either ccache_accept or ccache_reject.
-The other fields, however, (which are only relevant to domain records) may also
-contain ccache_unknown if that particular test has not been done.
+If a record exists, the result field is either ccache_accept or ccache_reject,
+or, for a domain record only, ccache_reject_mfnull when MAIL FROM:<> was
+rejected. The other fields, however, (which are only relevant to domain
+records) may also contain ccache_unknown if that particular test has not been
+done.
 
 Originally, there was only one structure, used for both types. However, it got
 expanded for domain records, so it got split. To make it possible for Exim to
diff --git a/src/src/macros.h b/src/src/macros.h
index 59c05e697..0b0215bdc 100644
--- a/src/src/macros.h
+++ b/src/src/macros.h
@@ -1,4 +1,4 @@
-/* $Cambridge: exim/src/src/macros.h,v 1.26 2006/06/28 16:00:24 ph10 Exp $ */
+/* $Cambridge: exim/src/src/macros.h,v 1.27 2006/09/05 13:24:10 ph10 Exp $ */
 
 /*************************************************
 *     Exim - an Internet mail transport agent    *
@@ -626,7 +626,8 @@ enum { v_none, v_sender, v_recipient, v_expn };
 
 #define ccache_unknown         0       /* test hasn't been done */
 #define ccache_accept          1
-#define ccache_reject          2
+#define ccache_reject          2       /* All rejections except */
+#define ccache_reject_mfnull   3       /* MAIL FROM:<> was rejected */
 
 /* Options for lookup functions */
 
diff --git a/src/src/verify.c b/src/src/verify.c
index 841285fdc..783378946 100644
--- a/src/src/verify.c
+++ b/src/src/verify.c
@@ -1,4 +1,4 @@
-/* $Cambridge: exim/src/src/verify.c,v 1.37 2006/06/30 15:36:08 ph10 Exp $ */
+/* $Cambridge: exim/src/src/verify.c,v 1.38 2006/09/05 13:24:10 ph10 Exp $ */
 
 /*************************************************
 *     Exim - an Internet mail transport agent    *
@@ -148,6 +148,7 @@ BOOL callout_no_cache = (options & vopt_callout_no_cache) != 0;
 BOOL callout_random = (options & vopt_callout_random) != 0;
 
 int yield = OK;
+int old_domain_cache_result = ccache_accept;
 BOOL done = FALSE;
 uschar *address_key;
 uschar *from_address;
@@ -228,10 +229,18 @@ if (dbm_file != NULL)
 
   if (cache_record != NULL)
     {
-    /* If an early command (up to and including MAIL FROM:<>) was rejected,
-    there is no point carrying on. The callout fails. */
-
-    if (cache_record->result == ccache_reject)
+    /* In most cases, if an early command (up to and including MAIL FROM:<>)
+    was rejected, there is no point carrying on. The callout fails. However, if
+    we are doing a recipient verification with use_sender or use_postmaster
+    set, a previous failure of MAIL FROM:<> doesn't count, because this time we
+    will be using a non-empty sender. We have to remember this situation so as
+    not to disturb the cached domain value if this whole verification succeeds
+    (we don't want it turning into "accept"). */
+
+    old_domain_cache_result = cache_record->result;
+
+    if (cache_record->result == ccache_reject ||
+         (*from_address == 0 && cache_record->result == ccache_reject_mfnull))
       {
       setflag(addr, af_verify_nsfail);
       HDEBUG(D_verify)
@@ -462,50 +471,73 @@ for (host = host_list; host != NULL && !done; host = host->next)
     continue;
     }
 
-  /* Wait for initial response, and then run the initial SMTP commands. The
-  smtp_write_command() function leaves its command in big_buffer. This is
-  used in error responses. Initialize it in case the connection is
-  rejected. */
+  /* Wait for initial response, and send HELO. The smtp_write_command()
+  function leaves its command in big_buffer. This is used in error responses.
+  Initialize it in case the connection is rejected. */
 
   Ustrcpy(big_buffer, "initial connection");
 
   done =
     smtp_read_response(&inblock, responsebuffer, sizeof(responsebuffer),
       '2', callout) &&
-
     smtp_write_command(&outblock, FALSE, "%s %s\r\n", helo,
       smtp_active_hostname) >= 0 &&
     smtp_read_response(&inblock, responsebuffer, sizeof(responsebuffer),
-      '2', callout) &&
+      '2', callout);
 
+  /* Failure to accept HELO is cached; this blocks the whole domain for all
+  senders. I/O errors and defer responses are not cached. */
+
+  if (!done)
+    {
+    *failure_ptr = US"mail";     /* At or before MAIL */
+    if (errno == 0 && responsebuffer[0] == '5')
+      {
+      setflag(addr, af_verify_nsfail);
+      new_domain_record.result = ccache_reject;
+      }
+    }
+
+  /* Send the MAIL command */
+
+  else done =
     smtp_write_command(&outblock, FALSE, "MAIL FROM:<%s>\r\n",
       from_address) >= 0 &&
     smtp_read_response(&inblock, responsebuffer, sizeof(responsebuffer),
       '2', callout);
 
-  /* If the host gave an initial error, or does not accept HELO or MAIL
-  FROM:<>, arrange to cache this information, but don't record anything for an
-  I/O error or a defer. Do not cache rejections when a non-empty sender has
-  been used, because that blocks the whole domain for all senders. */
+  /* If the host does not accept MAIL FROM:<>, arrange to cache this
+  information, but again, don't record anything for an I/O error or a defer. Do
+  not cache rejections of MAIL when a non-empty sender has been used, because
+  that blocks the whole domain for all senders. */
 
   if (!done)
     {
-    *failure_ptr = US"mail";
+    *failure_ptr = US"mail";     /* At or before MAIL */
     if (errno == 0 && responsebuffer[0] == '5')
       {
       setflag(addr, af_verify_nsfail);
-      if (from_address[0] == 0) new_domain_record.result = ccache_reject;
+      if (from_address[0] == 0)
+        new_domain_record.result = ccache_reject_mfnull;
       }
     }
 
   /* Otherwise, proceed to check a "random" address (if required), then the
   given address, and the postmaster address (if required). Between each check,
   issue RSET, because some servers accept only one recipient after MAIL
-  FROM:<>. */
+  FROM:<>.
+
+  Before doing this, set the result in the domain cache record to "accept",
+  unless its previous value was ccache_reject_mfnull. In that case, the domain
+  rejects MAIL FROM:<> and we want to continue to remember that. When that is
+  the case, we have got here only in the case of a recipient verification with
+  a non-null sender. */
 
   else
     {
-    new_domain_record.result = ccache_accept;
+    new_domain_record.result =
+      (old_domain_cache_result == ccache_reject_mfnull)?
+        ccache_reject_mfnull: ccache_accept;
 
     /* Do the random local part check first */
 
@@ -685,7 +717,7 @@ However, there may be domain-specific information to cache in both cases.
 The value of the result field in the new_domain record is ccache_unknown if
 there was an error before or with MAIL FROM:, and errno was not zero,
 implying some kind of I/O error. We don't want to write the cache in that case.
-Otherwise the value is ccache_accept or ccache_reject. */
+Otherwise the value is ccache_accept, ccache_reject, or ccache_reject_mfnull. */
 
 if (!callout_no_cache && new_domain_record.result != ccache_unknown)
   {
diff --git a/src/src/version.c b/src/src/version.c
index 179ebaa90..e83905eb4 100644
--- a/src/src/version.c
+++ b/src/src/version.c
@@ -1,4 +1,4 @@
-/* $Cambridge: exim/src/src/version.c,v 1.17 2006/06/27 13:39:24 ph10 Exp $ */
+/* $Cambridge: exim/src/src/version.c,v 1.18 2006/09/05 13:24:10 ph10 Exp $ */
 
 /*************************************************
 *     Exim - an Internet mail transport agent    *
@@ -12,7 +12,7 @@
 #include "exim.h"
 
 
-#define THIS_VERSION  "4.63"
+#define THIS_VERSION  "4.64"
 
 
 /* The header file cnumber.h contains a single line containing the
diff --git a/test/confs/0538 b/test/confs/0538
new file mode 100644
index 000000000..7f89ccc6a
--- /dev/null
+++ b/test/confs/0538
@@ -0,0 +1,43 @@
+# Exim test configuration 0538
+
+exim_path = EXIM_PATH
+host_lookup_order = bydns
+primary_hostname = myhost.test.ex
+rfc1413_query_timeout = 0s
+spool_directory = DIR/spool
+log_file_path = DIR/spool/log/%slog
+gecos_pattern = ""
+gecos_name = CALLER_NAME
+
+# ----- Main settings -----
+
+acl_smtp_mail = check_mail
+acl_smtp_rcpt = check_rcpt
+
+
+# ----- ACLs -----
+
+begin acl
+
+check_mail:
+  accept  sender_domains = broken.example
+          endpass
+          verify = sender/callout
+  accept
+
+check_rcpt:
+  accept  verify = recipient/callout=use_sender
+
+
+# ----- Routers -----
+
+begin routers
+
+r1:
+  driver = manualroute
+  route_list = * "<= 127.0.0.1:PORT_S"
+  self = send
+  verify_only
+
+
+# End
diff --git a/test/log/0538 b/test/log/0538
new file mode 100644
index 000000000..8b63bdb9b
--- /dev/null
+++ b/test/log/0538
@@ -0,0 +1,4 @@
+1999-03-02 09:44:33 U=CALLER sender verify fail for <userx@broken.example>: response to "MAIL FROM:<>" from 127.0.0.1 [127.0.0.1] was: 550 I'm misconfigured
+1999-03-02 09:44:33 U=CALLER rejected MAIL <userx@broken.example>: Sender verify failed
+1999-03-02 09:44:33 U=CALLER sender verify fail for <userx@broken.example>
+1999-03-02 09:44:33 U=CALLER rejected MAIL <userx@broken.example>: Sender verify failed
diff --git a/test/rejectlog/0538 b/test/rejectlog/0538
new file mode 100644
index 000000000..8b63bdb9b
--- /dev/null
+++ b/test/rejectlog/0538
@@ -0,0 +1,4 @@
+1999-03-02 09:44:33 U=CALLER sender verify fail for <userx@broken.example>: response to "MAIL FROM:<>" from 127.0.0.1 [127.0.0.1] was: 550 I'm misconfigured
+1999-03-02 09:44:33 U=CALLER rejected MAIL <userx@broken.example>: Sender verify failed
+1999-03-02 09:44:33 U=CALLER sender verify fail for <userx@broken.example>
+1999-03-02 09:44:33 U=CALLER rejected MAIL <userx@broken.example>: Sender verify failed
diff --git a/test/runtest b/test/runtest
index 992f02915..66a0197b9 100755
--- a/test/runtest
+++ b/test/runtest
@@ -1,6 +1,6 @@
 #! /usr/bin/perl -w
 
-# $Cambridge: exim/test/runtest,v 1.13 2006/07/26 14:39:13 ph10 Exp $
+# $Cambridge: exim/test/runtest,v 1.14 2006/09/05 13:24:10 ph10 Exp $
 
 ###############################################################################
 # This is the controlling script for the "new" test suite for Exim. It should #
@@ -23,7 +23,7 @@ use Socket;
 
 # Start by initializing some global variables
 
-$testversion = "4.63 (24-Jul-06)";
+$testversion = "4.64 (05-Sep-06)";
 
 $cf = "bin/cf";
 $cr = "\r";
diff --git a/test/scripts/0000-Basic/0538 b/test/scripts/0000-Basic/0538
new file mode 100644
index 000000000..c464f3bf1
--- /dev/null
+++ b/test/scripts/0000-Basic/0538
@@ -0,0 +1,39 @@
+# callout for recipient/use_sender after mail from:<> rejection
+need_ipv4
+#
+# Do a sender address verify that rejects MAIL FROM:<>
+server PORT_S
+220 Welcome
+HELO
+250 Hi
+MAIL FROM
+550 I'm misconfigured
+QUIT
+221 Bye
+****
+exim -bs
+mail from:<userx@broken.example>
+quit
+****
+# Now do a recipient verify for the same domain, with use_sender
+server PORT_S
+220 Welcome
+HELO
+250 Hi
+MAIL FROM
+250 OK
+RCPT TO
+250 OK
+QUIT
+221 Bye
+****
+exim -bs
+mail from:<userx@ok.example>
+rcpt to:<usery@broken.example>
+quit
+****
+# A final check that the cache works for sender address
+exim -bs
+mail from:<userx@broken.example>
+quit
+****
diff --git a/test/stderr/0376 b/test/stderr/0376
index f51553220..3a6367578 100644
--- a/test/stderr/0376
+++ b/test/stderr/0376
@@ -143,7 +143,7 @@ Connecting to 127.0.0.1 [127.0.0.1]:1224 ... connected
   SMTP<< 550 REJECT MAIL FROM
   SMTP>> QUIT
 wrote callout cache domain record:
-  result=2 postmaster=0 random=0
+  result=3 postmaster=0 random=0
 LOG: MAIN REJECT
   H=[V4NET.0.0.1] U=root sender verify fail for <ok@localhost>: response to "MAIL FROM:<>" from 127.0.0.1 [127.0.0.1] was: 550 REJECT MAIL FROM
 LOG: MAIN REJECT
diff --git a/test/stdout/0538 b/test/stdout/0538
new file mode 100644
index 000000000..c7bd59226
--- /dev/null
+++ b/test/stdout/0538
@@ -0,0 +1,50 @@
+220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000

+550-Callback setup failed while verifying <userx@broken.example>

+550-Called:   127.0.0.1

+550-Sent:     MAIL FROM:<>

+550-Response: 550 I'm misconfigured

+550-The initial connection, or a HELO or MAIL FROM:<> command was

+550-rejected. Refusing MAIL FROM:<> does not help fight spam, disregards

+550-RFC requirements, and stops you from receiving standard bounce

+550-messages. This host does not accept mail from domains whose servers

+550-refuse bounces.

+550 Sender verify failed

+221 myhost.test.ex closing connection

+220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000

+250 OK

+250 Accepted

+221 myhost.test.ex closing connection

+220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000

+550-Callback setup failed while verifying <userx@broken.example>

+550-(result of an earlier callout reused).

+550-The initial connection, or a HELO or MAIL FROM:<> command was

+550-rejected. Refusing MAIL FROM:<> does not help fight spam, disregards

+550-RFC requirements, and stops you from receiving standard bounce

+550-messages. This host does not accept mail from domains whose servers

+550-refuse bounces.

+550 Sender verify failed

+221 myhost.test.ex closing connection

+
+******** SERVER ********
+Listening on port 1224 ... 
+Connection request from [127.0.0.1]
+220 Welcome
+HELO myhost.test.ex
+250 Hi
+MAIL FROM:<>
+550 I'm misconfigured
+QUIT
+221 Bye
+End of script
+Listening on port 1224 ... 
+Connection request from [127.0.0.1]
+220 Welcome
+HELO myhost.test.ex
+250 Hi
+MAIL FROM:<userx@ok.example>
+250 OK
+RCPT TO:<usery@broken.example>
+250 OK
+QUIT
+221 Bye
+End of script