Merge branch 'master' into 4.next

14 files changed, 198 insertions, 28 deletions

diff --git a/src/OS/os.h-Linux b/src/OS/os.h-Linux
index cc1f3cab2..f6d35772b 100644
--- a/src/OS/os.h-Linux
+++ b/src/OS/os.h-Linux
@@ -79,6 +79,7 @@ then change the 0 to 1 in the next block. */
 #if defined(TCP_FASTOPEN) && !defined(MSG_FASTOPEN)
 # define MSG_FASTOPEN 0x20000000
 #endif
+#define EXIM_HAVE_TCPI_UNACKED
 
 
 /* End */
diff --git a/src/src/deliver.c b/src/src/deliver.c
index 73921980e..648c63d69 100644
--- a/src/src/deliver.c
+++ b/src/src/deliver.c
@@ -3535,7 +3535,8 @@ while (!done)
       break;
 
     case 'T':
-      setflag(addr, af_tcp_fastopen);
+      setflag(addr, af_tcp_fastopen_conn);
+      if (*subid > '0') setflag(addr, af_tcp_fastopen);
       break;
 
     case 'D':
@@ -4837,8 +4838,9 @@ all pipes, so I do not see a reason to use non-blocking IO here
       if (testflag(addr, af_chunking_used))
 	rmt_dlv_checked_write(fd, 'K', '0', NULL, 0);
 
-      if (testflag(addr, af_tcp_fastopen))
-	rmt_dlv_checked_write(fd, 'T', '0', NULL, 0);
+      if (testflag(addr, af_tcp_fastopen_conn))
+	rmt_dlv_checked_write(fd, 'T',
+	  testflag(addr, af_tcp_fastopen) ? '1' : '0', NULL, 0);
 
       memcpy(big_buffer, &addr->dsn_aware, sizeof(addr->dsn_aware));
       rmt_dlv_checked_write(fd, 'D', '0', big_buffer, sizeof(addr->dsn_aware));
diff --git a/src/src/globals.c b/src/src/globals.c
index 57041fc4e..2a53835e9 100644
--- a/src/src/globals.c
+++ b/src/src/globals.c
@@ -1421,7 +1421,7 @@ blob	tcp_fastopen_nodata    = { .data = NULL, .len = 0 };
 BOOL    tcp_in_fastopen        = FALSE;
 BOOL    tcp_in_fastopen_logged = FALSE;
 BOOL    tcp_nodelay            = TRUE;
-BOOL    tcp_out_fastopen       = FALSE;
+int     tcp_out_fastopen       = 0;
 BOOL    tcp_out_fastopen_logged= FALSE;
 #ifdef USE_TCP_WRAPPERS
 uschar *tcp_wrappers_daemon_name = US TCP_WRAPPERS_DAEMON_NAME;
diff --git a/src/src/globals.h b/src/src/globals.h
index 2957587b0..62336c275 100644
--- a/src/src/globals.h
+++ b/src/src/globals.h
@@ -923,10 +923,10 @@ extern BOOL    system_filtering;       /* TRUE when running system filter */
 
 extern BOOL    tcp_fastopen_ok;	       /* appears to be supported by kernel */
 extern blob    tcp_fastopen_nodata;    /* for zero-data TFO connect requests */
-extern BOOL    tcp_in_fastopen;	       /* conn used fastopen */
+extern BOOL    tcp_in_fastopen;	       /* conn usefully used fastopen */
 extern BOOL    tcp_in_fastopen_logged; /* one-time logging */
 extern BOOL    tcp_nodelay;            /* Controls TCP_NODELAY on daemon */
-extern BOOL    tcp_out_fastopen;       /* conn used fastopen */
+extern int     tcp_out_fastopen;       /* 0: no  1: conn used  2: useful */
 extern BOOL    tcp_out_fastopen_logged; /* one-time logging */
 #ifdef USE_TCP_WRAPPERS
 extern uschar *tcp_wrappers_daemon_name; /* tcpwrappers daemon lookup name */
diff --git a/src/src/ip.c b/src/src/ip.c
index 266eaf414..e11aef985 100644
--- a/src/src/ip.c
+++ b/src/src/ip.c
@@ -236,23 +236,29 @@ if (fastopen)
   {
   if ((rc = sendto(sock, fastopen->data, fastopen->len,
 		    MSG_FASTOPEN | MSG_DONTWAIT, s_ptr, s_len)) >= 0)
+	/* seen for with-data, experimental TFO option, with-cookie case */
+	/* seen for with-data, proper TFO opt, with-cookie case */
     {
-    DEBUG(D_transport|D_v)
-      debug_printf("TCP_FASTOPEN mode connection, with data\n");
-    tcp_out_fastopen = TRUE;
+    DEBUG(D_transport|D_v) debug_printf("TFO mode connection attempt, %s data\n",
+      fastopen->len > 0 ? "with"  : "no");
+    tcp_out_fastopen = fastopen->len > 0 ?  2 : 1;
     }
-  else if (errno == EINPROGRESS)	/* expected for nonready peer */
-    {
+  else if (errno == EINPROGRESS)	/* expected if we had no cookie for peer */
+	/* seen for no-data, proper TFO option, both cookie-request and with-cookie cases */
+	/*  apparently no visibility of the diffference at this point */
+	/* seen for with-data, proper TFO opt, cookie-req */
+	/*   with netwk delay, post-conn tcp_info sees unacked 1 for R, 2 for C; code in smtp_out.c */
+	/* ? older Experimental TFO option behaviour ? */
+    {					/* queue unsent data */
+    DEBUG(D_transport|D_v) debug_printf("TFO mode sendto, %s data: EINPROGRESS\n",
+      fastopen->len > 0 ? "with"  : "no");
     if (!fastopen->data)
       {
-      DEBUG(D_transport|D_v)
-	debug_printf("TCP_FASTOPEN mode connection, no data\n");
-      tcp_out_fastopen = TRUE;
+      tcp_out_fastopen = 1;		/* we tried; unknown if useful yet */
       rc = 0;
       }
-    else if (  (rc = send(sock, fastopen->data, fastopen->len, 0)) < 0
-	    && errno == EINPROGRESS)	/* expected for nonready peer */
-      rc = 0;
+    else
+      rc = send(sock, fastopen->data, fastopen->len, 0);
     }
   else if(errno == EOPNOTSUPP)
     {
diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c
index 36f685677..0f8d5599b 100644
--- a/src/src/smtp_in.c
+++ b/src/src/smtp_in.c
@@ -2346,7 +2346,7 @@ if (  getsockopt(fileno(smtp_out), IPPROTO_TCP, TCP_INFO, &tinfo, &len) == 0
    && tinfo.tcpi_state == TCP_SYN_RECV
    )
   {
-  DEBUG(D_receive) debug_printf("TCP_FASTOPEN mode connection\n");
+  DEBUG(D_receive) debug_printf("TCP_FASTOPEN mode connection (state TCP_SYN_RECV)\n");
   tcp_in_fastopen = TRUE;
   }
 # endif
diff --git a/src/src/smtp_out.c b/src/src/smtp_out.c
index db33ac66e..786f8b592 100644
--- a/src/src/smtp_out.c
+++ b/src/src/smtp_out.c
@@ -140,6 +140,60 @@ return TRUE;
 
 
 
+#ifdef TCP_FASTOPEN
+static void
+tfo_out_check(int sock)
+{
+# if defined(TCP_INFO) && defined(EXIM_HAVE_TCPI_UNACKED)
+struct tcp_info tinfo;
+socklen_t len = sizeof(tinfo);
+
+if (getsockopt(sock, IPPROTO_TCP, TCP_INFO, &tinfo, &len) == 0)
+  {
+  switch (tcp_out_fastopen)
+    {
+      /* This is a somewhat dubious detection method; totally undocumented so likely
+      to fail in future kernels.  There seems to be no documented way.  What we really
+      want to know is if the server sent smtp-banner data before our ACK of his SYN,ACK
+      hit him.  What this (possibly?) detects is whether we sent a TFO cookie with our
+      SYN, as distinct from a TFO request.  This gets a false-positive when the server
+      key is rotated; we send the old one (which this test sees) but the server returns
+      the new one and does not send its SMTP banner before we ACK his SYN,ACK.
+       To force that rotation case:
+       '# echo -n "00000000-00000000-00000000-0000000" >/proc/sys/net/ipv4/tcp_fastopen_key'
+      The kernel seems to be counting unack'd packets. */
+
+    case 1:
+      if (tinfo.tcpi_unacked > 1)
+	{
+	DEBUG(D_transport|D_v)
+	  debug_printf("TCP_FASTOPEN tcpi_unacked %d\n", tinfo.tcpi_unacked);
+	tcp_out_fastopen = 2;
+	}
+      break;
+
+#ifdef notdef		/* This seems to always fire, meaning that we cannot tell
+			whether the server accepted data we sent.  For now assume
+			that it did. */
+
+      /* If there was data-on-SYN but we had to retrasnmit it, declare no TFO */
+
+    case 2:
+      if (!(tinfo.tcpi_options & TCPI_OPT_SYN_DATA))
+	{
+	DEBUG(D_transport|D_v) debug_printf("TFO: had to retransmit\n");
+	tcp_out_fastopen = 0;
+	}
+      break;
+#endif
+    }
+
+  }
+# endif
+}
+#endif
+
+
 /* Arguments as for smtp_connect(), plus
   early_data	if non-NULL, data to be sent - preferably in the TCP SYN segment
 
@@ -158,6 +212,8 @@ int dscp_level;
 int dscp_option;
 int sock;
 int save_errno = 0;
+const blob * fastopen = NULL;
+
 
 #ifndef DISABLE_EVENT
 deliver_host_address = host->address;
@@ -207,8 +263,6 @@ requested some early-data then include that in the TFO request. */
 
 else
   {
-  const blob * fastopen = NULL;
-
 #ifdef TCP_FASTOPEN
   if (verify_check_given_host(&ob->hosts_try_fastopen, host) == OK)
     fastopen = early_data ? early_data : &tcp_fastopen_nodata;
@@ -254,6 +308,9 @@ else
     return -1;
     }
   if (ob->keepalive) ip_keepalive(sock, host->address, TRUE);
+#ifdef TCP_FASTOPEN
+  if (fastopen) tfo_out_check(sock);
+#endif
   return sock;
   }
 }
diff --git a/src/src/structs.h b/src/src/structs.h
index d8ac19ab8..c16899a0c 100644
--- a/src/src/structs.h
+++ b/src/src/structs.h
@@ -610,7 +610,8 @@ typedef struct address_item {
     BOOL af_cert_verified:1;		/* delivered with verified TLS cert */
     BOOL af_pass_message:1;		/* pass message in bounces */
     BOOL af_bad_reply:1;		/* filter could not generate autoreply */
-    BOOL af_tcp_fastopen:1;		/* delivery used TCP Fast Open */
+    BOOL af_tcp_fastopen_conn:1;	/* delivery connection used TCP Fast Open */
+    BOOL af_tcp_fastopen:1;		/* delivery usefuly used TCP Fast Open */
 #ifndef DISABLE_PRDR
     BOOL af_prdr_used:1;		/* delivery used SMTP PRDR */
 #endif
diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index 461b26c4a..14cfde72a 100644
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -2511,7 +2511,10 @@ for (addr = sx->first_addr, address_count = 0;
   uschar * rcpt_addr;
 
   if (tcp_out_fastopen && !tcp_out_fastopen_logged)
-    setflag(addr, af_tcp_fastopen);
+    {
+    setflag(addr, af_tcp_fastopen_conn);
+    if (tcp_out_fastopen > 1) setflag(addr, af_tcp_fastopen);
+    }
 
   addr->dsn_aware = sx->peer_offered & OPTION_DSN
     ? dsn_support_yes : dsn_support_no;
diff --git a/src/src/transports/smtp_socks.c b/src/src/transports/smtp_socks.c
index 1368849d6..c9907dd54 100644
--- a/src/src/transports/smtp_socks.c
+++ b/src/src/transports/smtp_socks.c
@@ -126,10 +126,12 @@ switch(method)
       for (i = 0; i<len; i++) debug_printf(" %02x", s[i]);
       debug_printf("\n");
       }
-    if (  send(fd, s, len, 0) < 0
-       || !fd_ready(fd, tmo-time(NULL))
-       || read(fd, s, 2) != 2
-       )
+    if (send(fd, s, len, 0) < 0)
+      return FAIL;
+#ifdef TCP_QUICKACK
+    (void) setsockopt(fd, IPPROTO_TCP, TCP_QUICKACK, US &off, sizeof(off));
+#endif
+    if (!fd_ready(fd, tmo-time(NULL)) || read(fd, s, 2) != 2)
       return FAIL;
     HDEBUG(D_transport|D_acl|D_v)
       debug_printf_indent("  SOCKS<< %02x %02x\n", s[0], s[1]);
@@ -315,6 +317,10 @@ HDEBUG(D_transport|D_acl|D_v) debug_printf_indent("  SOCKS>> 05 01 %02x\n", sob-
 
 /* expect method response */
 
+#ifdef TCP_QUICKACK
+(void) setsockopt(fd, IPPROTO_TCP, TCP_QUICKACK, US &off, sizeof(off));
+#endif
+
 if (  !fd_ready(fd, tmo-time(NULL))
    || read(fd, buf, 2) != 2
    )
diff --git a/test/log/4027 b/test/log/4027
index 3af3f325b..7df91966f 100644
--- a/test/log/4027
+++ b/test/log/4027
@@ -1,6 +1,9 @@
 1999-03-02 09:44:33 10HmaX-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local-esmtp S=sss
-1999-03-02 09:44:33 10HmaX-0005vi-00 => user_tfo@test.ex R=my_main_router T=my_smtp H=127.0.0.1 [127.0.0.1]:1224 PRX=[127.0.0.1]:1225 TFO C="250 accepted OK"
+1999-03-02 09:44:33 10HmaX-0005vi-00 => user_tfo@test.ex R=my_main_router T=my_smtp H=127.0.0.1 [127.0.0.1]:1224 PRX=[127.0.0.1]:1225 C="250 accepted OK"
 1999-03-02 09:44:33 10HmaX-0005vi-00 Completed
 1999-03-02 09:44:33 10HmaY-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local-esmtp S=sss
 1999-03-02 09:44:33 10HmaY-0005vi-00 => user_tfo@test.ex R=my_main_router T=my_smtp H=127.0.0.1 [127.0.0.1]:1224 PRX=[127.0.0.1]:1225 TFO C="250 accepted OK"
 1999-03-02 09:44:33 10HmaY-0005vi-00 Completed
+1999-03-02 09:44:33 10HmaZ-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local-esmtp S=sss
+1999-03-02 09:44:33 10HmaZ-0005vi-00 => user_tfo@test.ex R=my_main_router T=my_smtp H=127.0.0.1 [127.0.0.1]:1224 PRX=[127.0.0.1]:1225 TFO C="250 accepted OK"
+1999-03-02 09:44:33 10HmaZ-0005vi-00 Completed
diff --git a/test/scripts/1990-TCP-Fast-Open/1990 b/test/scripts/1990-TCP-Fast-Open/1990
index cbedd3622..3fffce2d4 100644
--- a/test/scripts/1990-TCP-Fast-Open/1990
+++ b/test/scripts/1990-TCP-Fast-Open/1990
@@ -7,7 +7,14 @@
 # A packet capture on the loopback interface will show the TFO
 # option on the SYN, but the fast-output SMTP banner will not
 # be seen unless you also deliberately emulate a long path:
-# 'sudo tc qdisc add dev lo root netem delay 100ms'
+# 'sudo tc qdisc add dev lo root netem delay 50ms'
+# You'll need kernel-modules-extra installed, or you get
+# an unhelpful error from RTNETLINK.
+# To tidy up:  'sudo tc qdisc delete dev lo root'
+#
+sudo perl
+system ("tc qdisc add dev lo root netem delay 50ms");
+****
 #
 # First time runs will see a TFO request option only; subsequent
 # ones should see the TFO cookie and fast-output SMTP banner
@@ -17,6 +24,10 @@
 # The client log => lint.ex  should have a "TFO" element.
 # Assuming this is the first run since boot, the a@test recipient will not.
 #
+sudo perl
+system ("ip tcp_metrics delete 127.0.0.1");
+****
+#
 # The server log <= line for b@test.ex  should have a "TFO" element, but
 # this will only be obtained when the above delay is inserted into the
 # loopback net path.
@@ -41,5 +52,10 @@ Testing
 ****
 sleep 3
 #
+#
+sudo perl
+system ("tc qdisc delete dev lo root");
+****
+#
 killdaemon
 no_msglog_check
diff --git a/test/scripts/4027-TFO-socks/4027 b/test/scripts/4027-TFO-socks/4027
index 533021cbf..3cfb43ee4 100644
--- a/test/scripts/4027-TFO-socks/4027
+++ b/test/scripts/4027-TFO-socks/4027
@@ -3,6 +3,48 @@
 munge loopback
 #
 #
+# TFO both clients and server, no cookie yet
+sudo perl
+system ("ip tcp_metrics delete 127.0.0.1");
+****
+#
+server -tfo PORT_D
+<<\x05\x01\x00
+>>\x05\x00
+<<\x05\x01\x00\x01\x7f\x00\x00\x01\x04\xc8
+>>\x05\x00\x00\x01\x7f\x00\x00\x01\xbe\xef
+220 Connected OK
+EHLO
+250-server id
+250
+MAIL FROM
+250
+RCPT TO
+250
+DATA
+354 go ahead
+.
+250 accepted OK
+QUIT
+250 bye
+****
+#
+#
+exim -odi -bs -DOPT=
+ehlo test.ex
+mail from:<>
+rcpt to:<user_tfo@test.ex>
+data
+Date: Fri, 17 Dec 2004 14:35:01 +0100
+Subject: message should be sent
+
+connection trying TFO via proxy; no cookie yet
+.
+quit
+****
+#
+#
+#
 # TFO client, not server
 server PORT_D
 <<\x05\x01\x00
@@ -79,5 +121,6 @@ via null-auth proxy
 quit
 ****
 #
+millisleep 500
 #
 # Ends
diff --git a/test/stdout/4027 b/test/stdout/4027
index 74837c4ac..79950e97f 100644
--- a/test/stdout/4027
+++ b/test/stdout/4027
@@ -20,6 +20,17 @@
 354 Enter message, ending with "." on a line by itself

 250 OK id=10HmaY-0005vi-00

 221 myhost.test.ex closing connection

+220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000

+250-myhost.test.ex Hello CALLER at test.ex

+250-SIZE 52428800

+250-8BITMIME

+250-PIPELINING

+250 HELP

+250 OK

+250 Accepted

+354 Enter message, ending with "." on a line by itself

+250 OK id=10HmaZ-0005vi-00

+221 myhost.test.ex closing connection

 
 ******** SERVER ********
 Listening on port 1225 ... 
@@ -37,6 +48,27 @@ MAIL FROM
 RCPT TO
 250
 DATA
+354 go ahead
+R
+250 accepted OK
+QUIT
+250 bye
+End of script
+Listening on port 1225 ... 
+Connection request from [ip4.ip4.ip4.ip4]
+<<\x05\x01\x00
+>>\x05\x00
+<<\x05\x01\x00\x01\x7f\x00\x00\x01\x04\xc8
+>>\x05\x00\x00\x01\x7f\x00\x00\x01\xbe\xef
+220 Connected OK
+EHLO
+250-server id
+250
+MAIL FROM
+250
+RCPT TO
+250
+DATA
 354 do me
 R
 250 accepted OK