Flush SMTP before callout (unless control=no_callout_flush).

10 files changed, 142 insertions, 10 deletions

diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog
index d041a49d3..0f6eeb392 100644
--- a/doc/doc-txt/ChangeLog
+++ b/doc/doc-txt/ChangeLog
@@ -1,4 +1,4 @@
-$Cambridge: exim/doc/doc-txt/ChangeLog,v 1.470 2007/02/06 11:11:39 ph10 Exp $
+$Cambridge: exim/doc/doc-txt/ChangeLog,v 1.471 2007/02/06 12:19:27 ph10 Exp $
 
 Change log file for Exim from version 4.21
 -------------------------------------------
@@ -77,6 +77,9 @@ PH/16 Add ${if forany and ${if forall.
 
 PH/17 Added dsn_from option to vary the From: line in DSNs.
 
+PH/18 Flush SMTP output before performing a callout, unless control =
+      no_callout_flush is set.
+
 
 Exim version 4.66
 -----------------
diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff
index 43be722b0..b05db458a 100644
--- a/doc/doc-txt/NewStuff
+++ b/doc/doc-txt/NewStuff
@@ -1,4 +1,4 @@
-$Cambridge: exim/doc/doc-txt/NewStuff,v 1.137 2007/02/06 11:11:39 ph10 Exp $
+$Cambridge: exim/doc/doc-txt/NewStuff,v 1.138 2007/02/06 12:19:27 ph10 Exp $
 
 New Features in Exim
 --------------------
@@ -233,8 +233,12 @@ Version 4.67
     time and date.
 
 11. Exim has been modified so that it flushes SMTP output before implementing
-    a delay in an ACL. This behaviour can be disabled by obeying control =
-    no_delay_flush at some earlier point.
+    a delay in an ACL. It also flushes the output before performing a callout,
+    as this can take a substantial time. These behaviours can be disabled by
+    obeying control = no_delay_flush or control = no_callout_flush,
+    respectively, at some earlier stage of the connection. The effect of the
+    new default behaviour is to disable the PIPELINING optimization in these
+    situations, in order to avoid unexpected timeouts in clients.
 
 12. There are two new expansion conditions that iterate over a list. They are
     called forany and forall, and they are used like this:
diff --git a/src/src/acl.c b/src/src/acl.c
index 0f484f21d..d0ed0a51c 100644
--- a/src/src/acl.c
+++ b/src/src/acl.c
@@ -1,4 +1,4 @@
-/* $Cambridge: exim/src/src/acl.c,v 1.71 2007/02/06 11:16:21 ph10 Exp $ */
+/* $Cambridge: exim/src/src/acl.c,v 1.72 2007/02/06 12:19:27 ph10 Exp $ */
 
 /*************************************************
 *     Exim - an Internet mail transport agent    *
@@ -188,7 +188,8 @@ enum {
   CONTROL_FAKEREJECT,
   CONTROL_NO_MULTILINE,
   CONTROL_NO_PIPELINING,
-  CONTROL_NO_DELAY_FLUSH
+  CONTROL_NO_DELAY_FLUSH,
+  CONTROL_NO_CALLOUT_FLUSH
 };
 
 /* ACL control names; keep in step with the table above! This list is used for
@@ -220,7 +221,8 @@ static uschar *controls[] = {
   US"fakereject",
   US"no_multiline",
   US"no_pipelining",
-  US"no_delay_flush"
+  US"no_delay_flush",
+  US"no_callout_flush"
 };
 
 /* Flags to indicate for which conditions/modifiers a string expansion is done
@@ -598,6 +600,9 @@ static unsigned int control_forbids[] = {
     (1<<ACL_WHERE_NOTSMTP_START),
 
   (1<<ACL_WHERE_NOTSMTP)|                          /* no_delay_flush */
+    (1<<ACL_WHERE_NOTSMTP_START),
+
+  (1<<ACL_WHERE_NOTSMTP)|                          /* no_callout_flush */
     (1<<ACL_WHERE_NOTSMTP_START)
 };
 
@@ -621,6 +626,7 @@ static control_def controls_list[] = {
   { US"caselower_local_part",    CONTROL_CASELOWER_LOCAL_PART, FALSE },
   { US"enforce_sync",            CONTROL_ENFORCE_SYNC, FALSE },
   { US"freeze",                  CONTROL_FREEZE, TRUE },
+  { US"no_callout_flush",        CONTROL_NO_CALLOUT_FLUSH, FALSE },
   { US"no_delay_flush",          CONTROL_NO_DELAY_FLUSH, FALSE },
   { US"no_enforce_sync",         CONTROL_NO_ENFORCE_SYNC, FALSE },
   { US"no_multiline_responses",  CONTROL_NO_MULTILINE, FALSE },
@@ -2615,6 +2621,10 @@ for (; cb != NULL; cb = cb->next)
       disable_delay_flush = TRUE;
       break;
 
+      case CONTROL_NO_CALLOUT_FLUSH:
+      disable_callout_flush = TRUE;
+      break;
+
       case CONTROL_FAKEDEFER:
       case CONTROL_FAKEREJECT:
       fake_response = (control_type == CONTROL_FAKEDEFER) ? DEFER : FAIL;
diff --git a/src/src/globals.c b/src/src/globals.c
index a8122d0bc..5f4ed83fd 100644
--- a/src/src/globals.c
+++ b/src/src/globals.c
@@ -1,4 +1,4 @@
-/* $Cambridge: exim/src/src/globals.c,v 1.70 2007/02/06 11:11:40 ph10 Exp $ */
+/* $Cambridge: exim/src/src/globals.c,v 1.71 2007/02/06 12:19:27 ph10 Exp $ */
 
 /*************************************************
 *     Exim - an Internet mail transport agent    *
@@ -499,6 +499,7 @@ int     demime_errorlevel      = 0;
 int     demime_ok              = 0;
 uschar *demime_reason          = NULL;
 #endif
+BOOL    disable_callout_flush  = FALSE;
 BOOL    disable_delay_flush    = FALSE;
 #ifdef ENABLE_DISABLE_FSYNC
 BOOL    disable_fsync          = FALSE;
diff --git a/src/src/globals.h b/src/src/globals.h
index 09674f22b..48b4e0ecc 100644
--- a/src/src/globals.h
+++ b/src/src/globals.h
@@ -1,4 +1,4 @@
-/* $Cambridge: exim/src/src/globals.h,v 1.50 2007/02/06 11:11:40 ph10 Exp $ */
+/* $Cambridge: exim/src/src/globals.h,v 1.51 2007/02/06 12:19:27 ph10 Exp $ */
 
 /*************************************************
 *     Exim - an Internet mail transport agent    *
@@ -278,6 +278,7 @@ extern int     demime_errorlevel;      /* Severity of MIME error */
 extern int     demime_ok;              /* Nonzero if message has been demimed */
 extern uschar *demime_reason;          /* Reason for broken MIME container */
 #endif
+extern BOOL    disable_callout_flush;  /* Don't flush before callouts */
 extern BOOL    disable_delay_flush;    /* Don't flush before "delay" in ACL */
 #ifdef ENABLE_DISABLE_FSYNC
 extern BOOL    disable_fsync;          /* Not for normal use */
diff --git a/src/src/verify.c b/src/src/verify.c
index 5ec90aaa8..d71f7e774 100644
--- a/src/src/verify.c
+++ b/src/src/verify.c
@@ -1,4 +1,4 @@
-/* $Cambridge: exim/src/src/verify.c,v 1.47 2007/01/30 15:10:59 ph10 Exp $ */
+/* $Cambridge: exim/src/src/verify.c,v 1.48 2007/02/06 12:19:27 ph10 Exp $ */
 
 /*************************************************
 *     Exim - an Internet mail transport agent    *
@@ -385,6 +385,14 @@ if (callout_overall < 0) callout_overall = 4 * callout;
 if (callout_connect < 0) callout_connect = callout;
 callout_start_time = time(NULL);
 
+/* Before doing a real callout, if this is an SMTP connection, flush the SMTP
+output because a callout might take some time. When PIPELINING is active and
+there are many recipients, the total time for doing lots of callouts can add up
+and cause the client to time out. So in this case we forgo the PIPELINING
+optimization. */
+
+if (smtp_out != NULL && !disable_callout_flush) mac_smtp_fflush();
+
 /* Now make connections to the hosts and do real callouts. The list of hosts
 is passed in as an argument. */
 
diff --git a/test/confs/0553 b/test/confs/0553
new file mode 100644
index 000000000..bdbc8df48
--- /dev/null
+++ b/test/confs/0553
@@ -0,0 +1,63 @@
+# Exim test configuration 0553
+
+DCF =
+SERVER =
+
+exim_path = EXIM_PATH
+host_lookup_order = bydns
+primary_hostname = myhost.test.ex
+rfc1413_query_timeout = 0s
+spool_directory = DIR/spool
+log_file_path = DIR/spool/log/SERVER%slog
+gecos_pattern = ""
+gecos_name = CALLER_NAME
+
+# ----- Main settings -----
+
+acl_smtp_rcpt = check_recipient
+
+queue_only
+
+# ----- ACL -----
+
+begin acl
+
+check_recipient:
+  # Callouts accepted with a delay
+  accept  senders = :
+          delay = 1s
+
+  # Non-callouts do the callout
+  accept  verify = recipient/callout
+          DCF
+
+
+# ----- Routers -----
+
+begin routers
+
+t1:
+  driver = manualroute
+  route_list = * 127.0.0.1 byname
+  self = send
+  transport = smtp
+
+
+# ----- Transports -----
+
+begin transports
+
+smtp:
+  driver = smtp
+  port = PORT_D
+  command_timeout = 2s
+
+
+# ----- Retry -----
+
+begin retry
+
+* * F,5d,10s
+
+
+# End
diff --git a/test/log/0553 b/test/log/0553
new file mode 100644
index 000000000..7c4feaef1
--- /dev/null
+++ b/test/log/0553
@@ -0,0 +1,18 @@
+1999-03-02 09:44:33 10HmaX-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss
+1999-03-02 09:44:33 Start queue run: pid=pppp -qf
+1999-03-02 09:44:33 10HmaX-0005vi-00 == userx1@test.ex R=t1 T=smtp defer (dd): Connection timed out: SMTP timeout while connected to 127.0.0.1 [127.0.0.1] after RCPT TO:<userx1@test.ex>
+1999-03-02 09:44:33 10HmaX-0005vi-00 == userx2@test.ex R=t1 T=smtp defer (dd): Connection timed out: SMTP timeout while connected to 127.0.0.1 [127.0.0.1] after RCPT TO:<userx1@test.ex>
+1999-03-02 09:44:33 10HmaX-0005vi-00 == userx3@test.ex R=t1 T=smtp defer (dd): Connection timed out: SMTP timeout while connected to 127.0.0.1 [127.0.0.1] after RCPT TO:<userx1@test.ex>
+1999-03-02 09:44:33 End queue run: pid=pppp -qf
+1999-03-02 09:44:33 Start queue run: pid=pppp -qf
+1999-03-02 09:44:33 10HmaX-0005vi-00 => userx1@test.ex R=t1 T=smtp H=127.0.0.1 [127.0.0.1]
+1999-03-02 09:44:33 10HmaX-0005vi-00 -> userx2@test.ex R=t1 T=smtp H=127.0.0.1 [127.0.0.1]
+1999-03-02 09:44:33 10HmaX-0005vi-00 -> userx3@test.ex R=t1 T=smtp H=127.0.0.1 [127.0.0.1]
+1999-03-02 09:44:33 10HmaX-0005vi-00 Completed
+1999-03-02 09:44:33 End queue run: pid=pppp -qf
+
+******** SERVER ********
+1999-03-02 09:44:33 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTP on port 1225
+1999-03-02 09:44:33 SMTP connection from localhost (myhost.test.ex) [127.0.0.1] lost while reading message data (header)
+1999-03-02 09:44:33 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTP on port 1225
+1999-03-02 09:44:33 10HmaY-0005vi-00 <= CALLER@myhost.test.ex H=localhost (myhost.test.ex) [127.0.0.1] P=esmtp S=sss id=E10HmaX-0005vi-00@myhost.test.ex
diff --git a/test/scripts/0000-Basic/0553 b/test/scripts/0000-Basic/0553
new file mode 100644
index 000000000..07dfb0bed
--- /dev/null
+++ b/test/scripts/0000-Basic/0553
@@ -0,0 +1,22 @@
+# callout autoflush
+need_ipv4
+#
+# Put a message on the queue (queue_only is set)
+exim userx1@test.ex userx2@test.ex userx3@test.ex
+****
+# This daemon is "old-style", without the flush
+exim -DSERVER=server -DDCF=control=no_callout_flush -bd -oX PORT_D
+****
+exim -qf
+****
+sleep 1
+killdaemon
+#
+# This daemon should flush before delaying
+exim -DSERVER=server -bd -oX PORT_D
+****
+exim -qf
+****
+sleep 1
+killdaemon
+no_msglog_check
diff --git a/test/stderr/0553 b/test/stderr/0553
new file mode 100644
index 000000000..045fadc9b
--- /dev/null
+++ b/test/stderr/0553
@@ -0,0 +1,2 @@
+
+******** SERVER ********