Bugs in temporary error message handling for smtp in lmtp mode.

9 files changed, 402 insertions, 17 deletions

diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog
index 8b8ec9113..71e3e5e7e 100644
--- a/doc/doc-txt/ChangeLog
+++ b/doc/doc-txt/ChangeLog
@@ -1,4 +1,4 @@
-$Cambridge: exim/doc/doc-txt/ChangeLog,v 1.319 2006/03/01 11:40:51 ph10 Exp $
+$Cambridge: exim/doc/doc-txt/ChangeLog,v 1.320 2006/03/01 16:07:16 ph10 Exp $
 
 Change log file for Exim from version 4.21
 -------------------------------------------
@@ -254,6 +254,24 @@ PH/50 When an Exim quota was set without a file count quota, and mailbox_size
 
 PH/51 Added ${time_eval: to convert Exim time strings into seconds.
 
+PH/52 Two bugs concerned with error handling when the smtp transport is
+      used in LMTP mode:
+
+      (i) Exim was not creating retry information for temporary errors given
+      for individual recipients after the DATA command when the smtp transport
+      was used in LMTP mode. This meant that they could be retried too
+      frequently, and not timed out correctly.
+
+      (ii) Exim was setting the flag that allows error details to be returned
+      for LMTP errors on RCPT commands, but not for LMTP errors for individual
+      recipients that were returned after the DATA command.
+
+PH/53 This is related to PH/52, but is more general: for any failing address,
+      when detailed error information was permitted to be returned to the
+      sender, but the error was temporary, then after the final timeout, only
+      "retry timeout exceeded" was returned. Now it returns the full error as
+      well as "retry timeout exceeded".
+
 
 Exim version 4.60
 -----------------
diff --git a/src/src/deliver.c b/src/src/deliver.c
index dda4897b9..0cb0132c4 100644
--- a/src/src/deliver.c
+++ b/src/src/deliver.c
@@ -1,4 +1,4 @@
-/* $Cambridge: exim/src/src/deliver.c,v 1.29 2006/02/21 16:24:19 ph10 Exp $ */
+/* $Cambridge: exim/src/src/deliver.c,v 1.30 2006/03/01 16:07:16 ph10 Exp $ */
 
 /*************************************************
 *     Exim - an Internet mail transport agent    *
@@ -4299,15 +4299,15 @@ introducing newlines. All lines are indented by 4; the initial printing
 position must be set before calling.
 
 This function used always to print the error. Nowadays we want to restrict it
-to cases such as SMTP errors from a remote host, and errors from :fail: and
-filter "fail". We no longer pass other information willy-nilly in bounce and
-warning messages. Text in user_message is always output; text in message only
-if the af_pass_message flag is set.
+to cases such as LMTP/SMTP errors from a remote host, and errors from :fail:
+and filter "fail". We no longer pass other information willy-nilly in bounce
+and warning messages. Text in user_message is always output; text in message
+only if the af_pass_message flag is set.
 
 Arguments:
   addr         the address
   f            the FILE to print on
-  s            some leading text
+  t            some leading text
 
 Returns:       nothing
 */
@@ -4316,14 +4316,11 @@ static void
 print_address_error(address_item *addr, FILE *f, uschar *t)
 {
 int count = Ustrlen(t);
-uschar *s = (addr->user_message != NULL)? addr->user_message : addr->message;
+uschar *s = testflag(addr, af_pass_message)? addr->message : NULL;
 
-if (addr->user_message != NULL)
-  s = addr->user_message;
-else
+if (s == NULL)
   {
-  if (!testflag(addr, af_pass_message) || addr->message == NULL) return;
-  s = addr->message;
+  if (addr->user_message != NULL) s = addr->user_message; else return;
   }
 
 fprintf(f, "\n    %s", t);
diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index 345fb951b..9b204e064 100644
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -1,4 +1,4 @@
-/* $Cambridge: exim/src/src/transports/smtp.c,v 1.23 2006/02/28 12:42:47 ph10 Exp $ */
+/* $Cambridge: exim/src/src/transports/smtp.c,v 1.24 2006/03/01 16:07:16 ph10 Exp $ */
 
 /*************************************************
 *     Exim - an Internet mail transport agent    *
@@ -1574,7 +1574,8 @@ if (!ok) ok = TRUE; else
 
       /* LMTP - if the response fails badly (e.g. timeout), use it for all the
       remaining addresses. Otherwise, it's a return code for just the one
-      address. */
+      address. For temporary errors, add a retry item for the address so that
+      it doesn't get tried again too soon. */
 
       if (lmtp)
         {
@@ -1584,7 +1585,14 @@ if (!ok) ok = TRUE; else
           if (errno != 0 || buffer[0] == 0) goto RESPONSE_FAILED;
           addr->message = string_sprintf("LMTP error after %s: %s",
             big_buffer, string_printing(buffer));
-          addr->transport_return = (buffer[0] == '5')? FAIL : DEFER;
+          setflag(addr, af_pass_message);   /* Allow message to go to user */
+          if (buffer[0] == '5')
+            addr->transport_return = FAIL;
+          else
+            {
+            addr->transport_return = DEFER;
+            retry_add_item(addr, addr->address_retry_key, 0);
+            }
           continue;
           }
         completed_address = TRUE;   /* NOW we can set this flag */
diff --git a/test/confs/0531 b/test/confs/0531
new file mode 100644
index 000000000..56f679ed8
--- /dev/null
+++ b/test/confs/0531
@@ -0,0 +1,58 @@
+# Exim test configuration 0531
+
+exim_path = EXIM_PATH
+host_lookup_order = bydns
+primary_hostname = myhost.test.ex
+rfc1413_query_timeout = 0s
+spool_directory = DIR/spool
+log_file_path = DIR/spool/log/%slog
+gecos_pattern = ""
+gecos_name = CALLER_NAME
+
+# ----- Main settings -----
+
+domainlist local_domains = test.ex : *.test.ex
+log_selector =  +sender_on_delivery
+
+
+# ----- Routers -----
+
+begin routers
+
+bounces:
+  driver = accept
+  senders = :
+  transport = t1
+
+smartuser:
+  driver = accept
+  retry_use_local_part
+  transport = lmtp
+
+
+# ----- Transports -----
+
+begin transports
+
+t1:
+  driver = appendfile
+  file = DIR/test-mail/$local_part
+  user = CALLER
+
+lmtp:
+  driver = smtp
+  allow_localhost
+  hosts = 127.0.0.1
+  port = PORT_S
+  protocol = LMTP
+
+
+# ----- Retry -----
+
+
+begin retry
+
+retry.test.ex * F,1s,1s
+
+
+# End
diff --git a/test/log/0531 b/test/log/0531
new file mode 100644
index 000000000..90293aae1
--- /dev/null
+++ b/test/log/0531
@@ -0,0 +1,25 @@
+1999-03-02 09:44:33 10HmaX-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss
+1999-03-02 09:44:33 10HmaX-0005vi-00 == userx@test.ex R=smartuser T=lmtp defer (0): LMTP error after DATA: 450 TEMPERROR
+1999-03-02 09:44:33 10HmaX-0005vi-00 => usery@test.ex F=<CALLER@myhost.test.ex> R=smartuser T=lmtp H=127.0.0.1 [127.0.0.1]
+1999-03-02 09:44:33 10HmaX-0005vi-00 ** userx@test.ex: retry timeout exceeded
+1999-03-02 09:44:33 10HmaY-0005vi-00 <= <> R=10HmaX-0005vi-00 U=EXIMUSER P=local S=sss
+1999-03-02 09:44:33 10HmaY-0005vi-00 => CALLER <CALLER@myhost.test.ex> F=<> R=bounces T=t1
+1999-03-02 09:44:33 10HmaY-0005vi-00 Completed
+1999-03-02 09:44:33 10HmaX-0005vi-00 Completed
+1999-03-02 09:44:33 10HmaZ-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss
+1999-03-02 09:44:33 10HmaZ-0005vi-00 == userx@retry.test.ex R=smartuser T=lmtp defer (0): LMTP error after DATA: 450 TEMPERROR
+1999-03-02 09:44:33 Start queue run: pid=pppp
+1999-03-02 09:44:33 10HmaZ-0005vi-00 == userx@retry.test.ex R=smartuser T=lmtp defer (0): LMTP error after DATA: 450 TEMPERROR
+1999-03-02 09:44:33 10HmaZ-0005vi-00 ** userx@retry.test.ex: retry timeout exceeded
+1999-03-02 09:44:33 10HmbA-0005vi-00 <= <> R=10HmaZ-0005vi-00 U=EXIMUSER P=local S=sss
+1999-03-02 09:44:33 10HmbA-0005vi-00 => CALLER <CALLER@myhost.test.ex> F=<> R=bounces T=t1
+1999-03-02 09:44:33 10HmbA-0005vi-00 Completed
+1999-03-02 09:44:33 10HmaZ-0005vi-00 Completed
+1999-03-02 09:44:33 End queue run: pid=pppp
+1999-03-02 09:44:33 10HmbB-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss
+1999-03-02 09:44:33 10HmbB-0005vi-00 == userx@test.ex R=smartuser T=lmtp defer (-44): SMTP error from remote mail server after RCPT TO:<userx@test.ex>: host 127.0.0.1 [127.0.0.1]: 450 TEMPERROR
+1999-03-02 09:44:33 10HmbB-0005vi-00 ** userx@test.ex: retry timeout exceeded
+1999-03-02 09:44:33 10HmbC-0005vi-00 <= <> R=10HmbB-0005vi-00 U=EXIMUSER P=local S=sss
+1999-03-02 09:44:33 10HmbC-0005vi-00 => CALLER <CALLER@myhost.test.ex> F=<> R=bounces T=t1
+1999-03-02 09:44:33 10HmbC-0005vi-00 Completed
+1999-03-02 09:44:33 10HmbB-0005vi-00 Completed
diff --git a/test/mail/0531.CALLER b/test/mail/0531.CALLER
new file mode 100644
index 000000000..a35a78259
--- /dev/null
+++ b/test/mail/0531.CALLER
@@ -0,0 +1,99 @@
+From MAILER-DAEMON Tue Mar 02 09:44:33 1999
+Received: from EXIMUSER by myhost.test.ex with local (Exim x.yz)
+	id 10HmaY-0005vi-00
+	for CALLER@myhost.test.ex; Tue, 2 Mar 1999 09:44:33 +0000
+X-Failed-Recipients: userx@test.ex
+Auto-Submitted: auto-replied
+From: Mail Delivery System <Mailer-Daemon@myhost.test.ex>
+To: CALLER@myhost.test.ex
+Subject: Mail delivery failed: returning message to sender
+Message-Id: <E10HmaY-0005vi-00@myhost.test.ex>
+Date: Tue, 2 Mar 1999 09:44:33 +0000
+
+This message was created automatically by mail delivery software.
+
+A message that you sent could not be delivered to one or more of its
+recipients. This is a permanent error. The following address(es) failed:
+
+  userx@test.ex
+    LMTP error after DATA: 450 TEMPERROR: retry timeout exceeded
+
+------ This is a copy of the message, including all the headers. ------
+
+Return-path: <CALLER@myhost.test.ex>
+Received: from CALLER by myhost.test.ex with local (Exim x.yz)
+	(envelope-from <CALLER@myhost.test.ex>)
+	id 10HmaX-0005vi-00; Tue, 2 Mar 1999 09:44:33 +0000
+Message-Id: <E10HmaX-0005vi-00@myhost.test.ex>
+From: CALLER_NAME <CALLER@myhost.test.ex>
+Date: Tue, 2 Mar 1999 09:44:33 +0000
+
+This is a test message.
+
+From MAILER-DAEMON Tue Mar 02 09:44:33 1999
+Received: from EXIMUSER by myhost.test.ex with local (Exim x.yz)
+	id 10HmbA-0005vi-00
+	for CALLER@myhost.test.ex; Tue, 2 Mar 1999 09:44:33 +0000
+X-Failed-Recipients: userx@retry.test.ex
+Auto-Submitted: auto-replied
+From: Mail Delivery System <Mailer-Daemon@myhost.test.ex>
+To: CALLER@myhost.test.ex
+Subject: Mail delivery failed: returning message to sender
+Message-Id: <E10HmbA-0005vi-00@myhost.test.ex>
+Date: Tue, 2 Mar 1999 09:44:33 +0000
+
+This message was created automatically by mail delivery software.
+
+A message that you sent could not be delivered to one or more of its
+recipients. This is a permanent error. The following address(es) failed:
+
+  userx@retry.test.ex
+    LMTP error after DATA: 450 TEMPERROR: retry timeout exceeded
+
+------ This is a copy of the message, including all the headers. ------
+
+Return-path: <CALLER@myhost.test.ex>
+Received: from CALLER by myhost.test.ex with local (Exim x.yz)
+	(envelope-from <CALLER@myhost.test.ex>)
+	id 10HmaZ-0005vi-00
+	for userx@retry.test.ex; Tue, 2 Mar 1999 09:44:33 +0000
+Message-Id: <E10HmaZ-0005vi-00@myhost.test.ex>
+From: CALLER_NAME <CALLER@myhost.test.ex>
+Date: Tue, 2 Mar 1999 09:44:33 +0000
+
+This is a test message.
+
+From MAILER-DAEMON Tue Mar 02 09:44:33 1999
+Received: from EXIMUSER by myhost.test.ex with local (Exim x.yz)
+	id 10HmbC-0005vi-00
+	for CALLER@myhost.test.ex; Tue, 2 Mar 1999 09:44:33 +0000
+X-Failed-Recipients: userx@test.ex
+Auto-Submitted: auto-replied
+From: Mail Delivery System <Mailer-Daemon@myhost.test.ex>
+To: CALLER@myhost.test.ex
+Subject: Mail delivery failed: returning message to sender
+Message-Id: <E10HmbC-0005vi-00@myhost.test.ex>
+Date: Tue, 2 Mar 1999 09:44:33 +0000
+
+This message was created automatically by mail delivery software.
+
+A message that you sent could not be delivered to one or more of its
+recipients. This is a permanent error. The following address(es) failed:
+
+  userx@test.ex
+    SMTP error from remote mail server after RCPT TO:<userx@test.ex>:
+  host 127.0.0.1 [127.0.0.1]: 450 TEMPERROR: retry timeout exceeded
+
+------ This is a copy of the message, including all the headers. ------
+
+Return-path: <CALLER@myhost.test.ex>
+Received: from CALLER by myhost.test.ex with local (Exim x.yz)
+	(envelope-from <CALLER@myhost.test.ex>)
+	id 10HmbB-0005vi-00
+	for userx@test.ex; Tue, 2 Mar 1999 09:44:33 +0000
+Message-Id: <E10HmbB-0005vi-00@myhost.test.ex>
+From: CALLER_NAME <CALLER@myhost.test.ex>
+Date: Tue, 2 Mar 1999 09:44:33 +0000
+
+This is a test message.
+
diff --git a/test/scripts/0000-Basic/0215 b/test/scripts/0000-Basic/0215
index 47f1f1cc8..066612047 100644
--- a/test/scripts/0000-Basic/0215
+++ b/test/scripts/0000-Basic/0215
@@ -1,4 +1,4 @@
-# LMTP over TCP/IP (with log_sender_on_delivery)
+# LMTP over TCP/IP
 need_ipv4
 #
 server PORT_S
diff --git a/test/scripts/0000-Basic/0531 b/test/scripts/0000-Basic/0531
new file mode 100644
index 000000000..5e89595b7
--- /dev/null
+++ b/test/scripts/0000-Basic/0531
@@ -0,0 +1,85 @@
+# LMTP over TCP/IP - temporary error handling
+need_ipv4
+# This one has no retry time, so will be bounced immediately.
+#
+server PORT_S
+220 ESMTP
+LHLO
+250-OK
+250 HELP
+MAIL FROM:
+250 Sender OK
+RCPT TO:
+250 Receiver OK
+RCPT TO:
+250 Receiver OK
+DATA
+354 Send it
+.
+450 TEMPERROR
+250 OK
+QUIT
+250 OK
+****
+exim -odi userx@test.ex usery@test.ex
+This is a test message.
+****
+# This one has a retry time, so will be deferred.
+#
+server PORT_S
+220 ESMTP
+LHLO
+250-OK
+250 HELP
+MAIL FROM:
+250 Sender OK
+RCPT TO:
+250 Receiver OK
+DATA
+354 Send it
+.
+450 TEMPERROR
+QUIT
+250 OK
+****
+exim -odi userx@retry.test.ex
+This is a test message.
+****
+sleep 1
+# Should by now have exceeded retry time.
+server PORT_S
+220 ESMTP
+LHLO
+250-OK
+250 HELP
+MAIL FROM:
+250 Sender OK
+RCPT TO:
+250 Receiver OK
+DATA
+354 Send it
+.
+450 TEMPERROR
+QUIT
+250 OK
+****
+exim -q
+****
+# This one gives a temporary error for RCPT, no retry
+#
+server PORT_S
+220 ESMTP
+LHLO
+250-OK
+250 HELP
+MAIL FROM:
+250 Sender OK
+RCPT TO:
+450 TEMPERROR
+QUIT
+250 OK
+****
+exim -odi userx@test.ex
+This is a test message.
+****
+no_msglog_check
diff --git a/test/stdout/0531 b/test/stdout/0531
new file mode 100644
index 000000000..01438bd35
--- /dev/null
+++ b/test/stdout/0531
@@ -0,0 +1,95 @@
+
+******** SERVER ********
+Listening on port 1224 ... 
+Connection request from [127.0.0.1]
+220 ESMTP
+LHLO myhost.test.ex
+250-OK
+250 HELP
+MAIL FROM:<CALLER@myhost.test.ex>
+250 Sender OK
+RCPT TO:<userx@test.ex>
+250 Receiver OK
+RCPT TO:<usery@test.ex>
+250 Receiver OK
+DATA
+354 Send it
+Received: from CALLER by myhost.test.ex with local (Exim x.yz)
+	(envelope-from <CALLER@myhost.test.ex>)
+	id 10HmaX-0005vi-00; Tue, 2 Mar 1999 09:44:33 +0000
+Message-Id: <E10HmaX-0005vi-00@myhost.test.ex>
+From: CALLER_NAME <CALLER@myhost.test.ex>
+Date: Tue, 2 Mar 1999 09:44:33 +0000
+
+This is a test message.
+.
+450 TEMPERROR
+250 OK
+QUIT
+250 OK
+End of script
+Listening on port 1224 ... 
+Connection request from [127.0.0.1]
+220 ESMTP
+LHLO myhost.test.ex
+250-OK
+250 HELP
+MAIL FROM:<CALLER@myhost.test.ex>
+250 Sender OK
+RCPT TO:<userx@retry.test.ex>
+250 Receiver OK
+DATA
+354 Send it
+Received: from CALLER by myhost.test.ex with local (Exim x.yz)
+	(envelope-from <CALLER@myhost.test.ex>)
+	id 10HmaZ-0005vi-00
+	for userx@retry.test.ex; Tue, 2 Mar 1999 09:44:33 +0000
+Message-Id: <E10HmaZ-0005vi-00@myhost.test.ex>
+From: CALLER_NAME <CALLER@myhost.test.ex>
+Date: Tue, 2 Mar 1999 09:44:33 +0000
+
+This is a test message.
+.
+450 TEMPERROR
+QUIT
+250 OK
+End of script
+Listening on port 1224 ... 
+Connection request from [127.0.0.1]
+220 ESMTP
+LHLO myhost.test.ex
+250-OK
+250 HELP
+MAIL FROM:<CALLER@myhost.test.ex>
+250 Sender OK
+RCPT TO:<userx@retry.test.ex>
+250 Receiver OK
+DATA
+354 Send it
+Received: from CALLER by myhost.test.ex with local (Exim x.yz)
+	(envelope-from <CALLER@myhost.test.ex>)
+	id 10HmaZ-0005vi-00
+	for userx@retry.test.ex; Tue, 2 Mar 1999 09:44:33 +0000
+Message-Id: <E10HmaZ-0005vi-00@myhost.test.ex>
+From: CALLER_NAME <CALLER@myhost.test.ex>
+Date: Tue, 2 Mar 1999 09:44:33 +0000
+
+This is a test message.
+.
+450 TEMPERROR
+QUIT
+250 OK
+End of script
+Listening on port 1224 ... 
+Connection request from [127.0.0.1]
+220 ESMTP
+LHLO myhost.test.ex
+250-OK
+250 HELP
+MAIL FROM:<CALLER@myhost.test.ex>
+250 Sender OK
+RCPT TO:<userx@test.ex>
+450 TEMPERROR
+QUIT
+250 OK
+End of script