MacOS: TCP Fast Open

11 files changed, 96 insertions, 29 deletions

diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff
index 10461aa05..cc9721ada 100644
--- a/doc/doc-txt/NewStuff
+++ b/doc/doc-txt/NewStuff
@@ -29,6 +29,8 @@ Version 4.92
 
  8. A "noutf8" debug option, for disabling the UTF-8 characters in debug output.
 
+ 9. TCP Fast Open support on MacOS.
+
 Version 4.91
 --------------
 
diff --git a/src/OS/os.h-Darwin b/src/OS/os.h-Darwin
index 67aeac913..4667689c9 100644
--- a/src/OS/os.h-Darwin
+++ b/src/OS/os.h-Darwin
@@ -51,10 +51,8 @@ in "man 2 getgroups". */
 #define _DARWIN_UNLIMITED_GETGROUPS
 #define EXIM_GROUPLIST_SIZE 64
 
-/* TCP_FASTOPEN support.  For the moment, claim there is none
-(the probe fails; unsure why).
-Sometime in the future need to investigate connectex(). */
-
-#define EXIM_TFO_PROBE
+/* TCP Fast Open: Darwin uses a connectex() call
+rather than a modified sendto() */
+#define EXIM_TFO_CONNECTX
 
 /* End */
diff --git a/src/src/daemon.c b/src/src/daemon.c
index ee9ddcc4f..1a15d46c0 100644
--- a/src/src/daemon.c
+++ b/src/src/daemon.c
@@ -1458,7 +1458,7 @@ if (f.daemon_listen && !f.inetd_wait_mode)
       else
         debug_printf("listening on %s port %d\n", ipa->address, ipa->port);
 
-#ifdef TCP_FASTOPEN
+#if defined(TCP_FASTOPEN) && !defined(__APPLE__)
     if (  f.tcp_fastopen_ok
        && setsockopt(listen_sockets[sk], IPPROTO_TCP, TCP_FASTOPEN,
 		    &smtp_connect_backlog, sizeof(smtp_connect_backlog)))
@@ -1471,7 +1471,19 @@ if (f.daemon_listen && !f.inetd_wait_mode)
     /* Start listening on the bound socket, establishing the maximum backlog of
     connections that is allowed. On success, continue to the next address. */
 
-    if (listen(listen_sockets[sk], smtp_connect_backlog) >= 0) continue;
+    if (listen(listen_sockets[sk], smtp_connect_backlog) >= 0)
+      {
+#if defined(TCP_FASTOPEN) && defined(__APPLE__)
+      if (  f.tcp_fastopen_ok
+	 && setsockopt(listen_sockets[sk], IPPROTO_TCP, TCP_FASTOPEN,
+		      &on, sizeof(on)))
+	{
+	DEBUG(D_any) debug_printf("setsockopt FASTOPEN: %s\n", strerror(errno));
+	f.tcp_fastopen_ok = FALSE;
+	}
+#endif
+      continue;
+      }
 
     /* Listening has failed. In an IPv6 environment, as for bind(), if listen()
     fails with the error EADDRINUSE and we are doing IPv4 wildcard listening
diff --git a/src/src/expand.c b/src/src/expand.c
index 5054e151b..49e09ecd8 100644
--- a/src/src/expand.c
+++ b/src/src/expand.c
@@ -5021,6 +5021,7 @@ while (*s != 0)
             port = ntohs(service_info->s_port);
             }
 
+	  /*XXX we trust that the request is idempotent.  Hmm. */
 	  fd = ip_connectedsocket(SOCK_STREAM, server_name, port, port,
 		  timeout, &host, &expand_string_message,
 		  do_tls ? NULL : &reqstr);
diff --git a/src/src/ip.c b/src/src/ip.c
index fa688be64..d601b46b3 100644
--- a/src/src/ip.c
+++ b/src/src/ip.c
@@ -196,7 +196,7 @@ Arguments:
   port        the remote port
   timeout     a timeout (zero for indefinite timeout)
   fastopen_blob    non-null iff TCP_FASTOPEN can be used; may indicate early-data to
-		be sent in SYN segment
+		be sent in SYN segment.  Any such data must be idempotent.
 
 Returns:      0 on success; -1 on failure, with errno set
 */
@@ -245,19 +245,19 @@ callout_address = string_sprintf("[%s]:%d", address, port);
 sigalrm_seen = FALSE;
 if (timeout > 0) ALARM(timeout);
 
-#if defined(TCP_FASTOPEN) && defined(MSG_FASTOPEN)
+#ifdef TCP_FASTOPEN
 /* TCP Fast Open, if the system has a cookie from a previous call to
 this peer, can send data in the SYN packet.  The peer can send data
 before it gets our ACK of its SYN,ACK - the latter is useful for
 the SMTP banner.  Other (than SMTP) cases of TCP connections can
-possibly use the data-on-syn, so support that too.
-
-This is a Linux implementation.  It might be useable on FreeBSD; I have
-not checked.  I think MacOS has a "connectx" call for this purpose,
-rather than using "sendto" ? */
+possibly use the data-on-syn, so support that too. */
 
 if (fastopen_blob && f.tcp_fastopen_ok)
   {
+# ifdef MSG_FASTOPEN
+  /* This is a Linux implementation.  It might be useable on FreeBSD; I have
+  not checked. */
+
   if ((rc = sendto(sock, fastopen_blob->data, fastopen_blob->len,
 		    MSG_FASTOPEN | MSG_DONTWAIT, s_ptr, s_len)) >= 0)
 	/* seen for with-data, experimental TFO option, with-cookie case */
@@ -292,9 +292,44 @@ if (fastopen_blob && f.tcp_fastopen_ok)
       debug_printf("Tried TCP Fast Open but apparently not enabled by sysctl\n");
     goto legacy_connect;
     }
+# endif
+# ifdef EXIM_TFO_CONNECTX
+  /* MacOS */
+  sa_endpoints_t ends = {
+    .sae_srcif = 0, .sae_srcaddr = NULL, .sae_srcaddrlen = 0,
+    .sae_dstaddr = s_ptr, .sae_dstaddrlen = s_len };
+  struct iovec iov = {
+    .iov_base = fastopen_blob->data, .iov_len = fastopen_blob->len };
+  size_t len;
+
+  if ((rc = connectx(sock, &ends, SAE_ASSOCID_ANY,
+	     CONNECT_DATA_IDEMPOTENT, &iov, 1, &len, NULL)) == 0)
+    {
+    DEBUG(D_transport|D_v)
+      debug_printf("TFO mode connection attempt to %s, %lu data\n",
+	address, (unsigned long)fastopen_blob->len);
+    tcp_out_fastopen = fastopen_blob->len > 0 ?  TFO_USED : TFO_ATTEMPTED;
+
+    if (len != fastopen_blob->len)
+      DEBUG(D_transport|D_v)
+	debug_printf(" only queued %lu data!\n", (unsigned long)len);
+    }
+  else if (errno == EINPROGRESS)
+    {
+    DEBUG(D_transport|D_v) debug_printf("TFO mode sendto, %s data: EINPROGRESS\n",
+      fastopen_blob->len > 0 ? "with"  : "no");
+    if (!fastopen_blob->data)
+      {
+      tcp_out_fastopen = TFO_ATTEMPTED;		/* we tried; unknown if useful yet */
+      rc = 0;
+      }
+    else	/* assume that no data was queued; block in send */
+      rc = send(sock, fastopen_blob->data, fastopen_blob->len, 0);
+    }
+# endif
   }
 else
-#endif
+#endif	/*TCP_FASTOPEN*/
   {
 legacy_connect:
   DEBUG(D_transport|D_v) if (fastopen_blob)
@@ -350,7 +385,7 @@ Arguments:
   connhost	if not NULL, host_item to be filled in with connection details
   errstr        pointer for allocated string on error
   fastopen_blob	with SOCK_STREAM, if non-null, request TCP Fast Open.
-		Additionally, optional early-data to send
+		Additionally, optional idempotent early-data to send
 
 Return:
   socket fd, or -1 on failure (having allocated an error string)
diff --git a/src/src/malware.c b/src/src/malware.c
index 069a54483..541ff3b3c 100644
--- a/src/src/malware.c
+++ b/src/src/malware.c
@@ -1573,6 +1573,7 @@ badseek:  err = errno;
 	   * on both connections (as one host could resolve to multiple ips) */
 	  for (;;)
 	    {
+	    /*XXX we trust that the cmd_str is ideempotent */
 	    if ((malware_daemon_ctx.sock = m_tcpsocket(cd->hostspec, cd->tcp_port,
 				    &connhost, &errstr, &cmd_str)) >= 0)
 	      {
diff --git a/src/src/smtp_out.c b/src/src/smtp_out.c
index ef2c9fdeb..c5eafbc57 100644
--- a/src/src/smtp_out.c
+++ b/src/src/smtp_out.c
@@ -194,7 +194,8 @@ if (getsockopt(sock, IPPROTO_TCP, TCP_INFO, &tinfo, &len) == 0)
 
 
 /* Arguments as for smtp_connect(), plus
-  early_data	if non-NULL, data to be sent - preferably in the TCP SYN segment
+  early_data	if non-NULL, idenmpotent data to be sent -
+		preferably in the TCP SYN segment
 
 Returns:      connected socket number, or -1 with errno set
 */
diff --git a/src/src/transports/smtp_socks.c b/src/src/transports/smtp_socks.c
index c7415c357..7d3a46230 100644
--- a/src/src/transports/smtp_socks.c
+++ b/src/src/transports/smtp_socks.c
@@ -297,6 +297,7 @@ for(;;)
   proxy.address = proxy.name = sob->proxy_host;
   proxy_af = Ustrchr(sob->proxy_host, ':') ? AF_INET6 : AF_INET;
 
+  /*XXX we trust that the method-select command is idempotent */
   if ((fd = smtp_sock_connect(&proxy, proxy_af, sob->port,
 	      interface, tb, sob->timeout, &early_data)) >= 0)
     {
diff --git a/src/src/verify.c b/src/src/verify.c
index 9aff78a9a..d14cb685e 100644
--- a/src/src/verify.c
+++ b/src/src/verify.c
@@ -2745,6 +2745,7 @@ qlen = snprintf(CS buffer, sizeof(buffer), "%d , %d\r\n",
 early_data.data = buffer;
 early_data.len = qlen;
 
+/*XXX we trust that the query is idempotent */
 if (ip_connect(ident_conn_ctx.sock, host_af, sender_host_address, port,
 		rfc1413_query_timeout, &early_data) < 0)
   {
diff --git a/test/scripts/1990-TCP-Fast-Open/1990 b/test/scripts/1990-TCP-Fast-Open/1990
index 04b41a7af..ec8e32c8f 100644
--- a/test/scripts/1990-TCP-Fast-Open/1990
+++ b/test/scripts/1990-TCP-Fast-Open/1990
@@ -13,6 +13,15 @@
 # an unhelpful error from RTNETLINK.
 # To tidy up:  'sudo tc qdisc delete dev lo root'
 #
+# MacOS:
+# The kernel seems to have TFO enabled both ways as default.
+# There is a net.inet.tcp.clear_tfocache parameter
+## sysctl -w foo-val
+#
+# For network delays there is something called 'Network Link Conditioner'
+# which might do the job.  But how to manipulate it?
+#
+#
 sudo perl
 system ("tc qdisc add dev lo root netem delay 50ms");
 ****
@@ -22,17 +31,14 @@ system ("tc qdisc add dev lo root netem delay 50ms");
 # (currently on a separate packet after the server SYN,ACK but before
 # the client ACK).
 #
-# The client log => lint.ex  should have a "TFO" element.
-# Assuming this is the first run since boot, the a@test recipient will not.
+# The client log => line  should have a "TFO" element.
+# The server log <= line for a@test.ex  should not.
 #
+# First clear any previously-obtained cookie:
 sudo perl
 system ("ip tcp_metrics delete 127.0.0.1");
 ****
 #
-# The server log <= line for b@test.ex  should have a "TFO" element, but
-# this will only be obtained when the above delay is inserted into the
-# loopback net path.
-#
 #
 #
 # FreeBSD: it looks like you have to compile a custom kernel, with
@@ -48,6 +54,10 @@ Testing
 ****
 sleep 3
 #
+# The server log <= line for b@test.ex  should have a "TFO" element, but
+# this will only be obtained when the above delay is inserted into the
+# loopback net path.
+#
 exim b@test.ex
 Testing
 ****
diff --git a/test/src/server.c b/test/src/server.c
index fe1c79f02..ba731625b 100644
--- a/test/src/server.c
+++ b/test/src/server.c
@@ -298,7 +298,7 @@ else
       printf("IPv6 socket creation failed: %s\n", strerror(errno));
       exit(1);
       }
-#ifdef TCP_FASTOPEN
+#if defined(TCP_FASTOPEN) && !defined(__APPLE__)
     if (tfo)
       {
       int backlog = 5;
@@ -330,7 +330,7 @@ else
       printf("IPv4 socket creation failed: %s\n", strerror(errno));
       exit(1);
       }
-#ifdef TCP_FASTOPEN
+#if defined(TCP_FASTOPEN) && !defined(__APPLE__)
     if (tfo)
       {
       int backlog = 5;
@@ -438,16 +438,21 @@ error because it means that the IPv6 socket will handle IPv4 connections. Don't
 output anything, because it will mess up the test output, which will be
 different for systems that do this and those that don't. */
 
-for (i = 0; i <= skn; i++)
+for (i = 0; i <= skn; i++) if (listen_socket[i] >= 0)
   {
-  if (listen_socket[i] >= 0 && listen(listen_socket[i], 5) < 0)
-    {
+  if (listen(listen_socket[i], 5) < 0)
     if (i != v4n || listen_socket[v6n] < 0 || errno != EADDRINUSE)
       {
       printf("listen() failed: %s\n", strerror(errno));
       exit(1);
       }
-    }
+
+#if defined(TCP_FASTOPEN) && defined(__APPLE__)
+  if (  tfo
+     && setsockopt(listen_socket[v4n], IPPROTO_TCP, TCP_FASTOPEN, &on, sizeof(on))
+     && debug)
+      printf("setsockopt TCP_FASTOPEN: %s\n", strerror(errno));
+#endif
   }