FreeBSD: better support for TFO

6 files changed, 143 insertions, 81 deletions

diff --git a/src/OS/os.c-FreeBSD b/src/OS/os.c-FreeBSD
index 4cc46c752..c0fd48df8 100644
--- a/src/OS/os.c-FreeBSD
+++ b/src/OS/os.c-FreeBSD
@@ -10,7 +10,7 @@ src/os.c file. */
 
 
 /*************
-* Sendfile   *
+Sendfile shim
 *************/
 
 ssize_t
@@ -23,4 +23,25 @@ if (sendfile(in, out, loff, cnt, NULL, &written, 0) < 0) return (ssize_t)-1;
 return (ssize_t)written;
 }
 
+/*************************************************
+TCP Fast Open:  check that the ioctl is accepted
+*************************************************/
+
+#ifndef COMPILE_UTILITY
+void
+tfo_probe(void)
+{
+# ifdef TCP_FASTOPEN
+int sock;
+
+if (  (sock = socket(AF_INET, SOCK_STREAM, 0)) >= 0
+   && setsockopt(sock, IPPROTO_TCP, TCP_FASTOPEN, &on, sizeof(on) >= 0)
+   )
+  f.tcp_fastopen_ok = TRUE;
+close(sock);
+# endif
+}
+#endif
+
+
 /* End of os.c-Linux */
diff --git a/src/OS/os.h-FreeBSD b/src/OS/os.h-FreeBSD
index 4f1c616f0..a15d47508 100644
--- a/src/OS/os.h-FreeBSD
+++ b/src/OS/os.h-FreeBSD
@@ -57,15 +57,9 @@ extern ssize_t os_sendfile(int, int, off_t *, size_t);
 
 /*******************/
 
-/* TCP_FASTOPEN support.  There does not seems to be a
-MSG_FASTOPEN defined yet... */
 #define EXIM_TFO_PROBE
+#define EXIM_TFO_FREEBSD
 
-#include <netinet/tcp.h>        /* for TCP_FASTOPEN */
-#include <sys/socket.h>         /* for MSG_FASTOPEN */
-#if defined(TCP_FASTOPEN) && !defined(MSG_FASTOPEN)
-# define MSG_FASTOPEN 0x20000000
-#endif
 
 /* for TCP state-variable values, for TFO logging */
 #include <netinet/tcp_fsm.h>
diff --git a/src/src/ip.c b/src/src/ip.c
index 70e3e2064..108c21d92 100644
--- a/src/src/ip.c
+++ b/src/src/ip.c
@@ -14,6 +14,12 @@ different places in the code where sockets are used. */
 #include "exim.h"
 
 
+#if defined(TCP_FASTOPEN)
+# if defined(MSG_FASTOPEN) || defined(EXIM_TFO_CONNECTX) || defined(EXIM_TFO_FREEBSD)
+#  define EXIM_SUPPORT_TFO
+# endif
+#endif
+
 /*************************************************
 *             Create a socket                    *
 *************************************************/
@@ -161,26 +167,6 @@ return bind(sock, (struct sockaddr *)&sin, s_len);
 
 
 /*************************************************
-*************************************************/
-
-#ifdef EXIM_TFO_PROBE
-void
-tfo_probe(void)
-{
-# ifdef TCP_FASTOPEN
-int sock, backlog = 5;
-
-if (  (sock = socket(SOCK_STREAM, AF_INET, 0)) < 0
-   && setsockopt(sock, IPPROTO_TCP, TCP_FASTOPEN, &backlog, sizeof(backlog))
-   )
-  f.tcp_fastopen_ok = TRUE;
-close(sock);
-# endif
-}
-#endif
-
-
-/*************************************************
 *        Connect socket to remote host           *
 *************************************************/
 
@@ -245,7 +231,7 @@ callout_address = string_sprintf("[%s]:%d", address, port);
 sigalrm_seen = FALSE;
 if (timeout > 0) ALARM(timeout);
 
-#if defined(TCP_FASTOPEN) && (defined(MSG_FASTOPEN) || defined(EXIM_TFO_CONNECTX))
+#ifdef EXIM_SUPPORT_TFO
 /* TCP Fast Open, if the system has a cookie from a previous call to
 this peer, can send data in the SYN packet.  The peer can send data
 before it gets our ACK of its SYN,ACK - the latter is useful for
@@ -255,8 +241,7 @@ possibly use the data-on-syn, so support that too. */
 if (fastopen_blob && f.tcp_fastopen_ok)
   {
 # ifdef MSG_FASTOPEN
-  /* This is a Linux implementation.  FreeBSD does not seem to have MSG_FASTOPEN so
-  how to get TFO is unknown. */
+  /* This is a Linux implementation. */
 
   if ((rc = sendto(sock, fastopen_blob->data, fastopen_blob->len,
 		    MSG_FASTOPEN | MSG_DONTWAIT, s_ptr, s_len)) >= 0)
@@ -292,8 +277,26 @@ if (fastopen_blob && f.tcp_fastopen_ok)
       debug_printf("Tried TCP Fast Open but apparently not enabled by sysctl\n");
     goto legacy_connect;
     }
-# endif
-# ifdef EXIM_TFO_CONNECTX
+
+# elif defined(EXIM_TFO_FREEBSD)
+  /* Re: https://people.freebsd.org/~pkelsey/tfo-tools/tfo-client.c */
+
+  if (setsockopt(sock, IPPROTO_TCP, TCP_FASTOPEN, &on, sizeof(on)) < 0)
+    {
+    DEBUG(D_transport)
+      debug_printf("Tried TCP Fast Open but apparently not enabled by sysctl\n");
+    goto legacy_connect;
+    }
+  if ((rc = sendto(sock, fastopen_blob->data, fastopen_blob->len, 0,
+		    s_ptr, s_len)) >= 0)
+    {
+    DEBUG(D_transport|D_v)
+      debug_printf(" TFO mode connection attempt to %s, %lu data\n",
+	address, (unsigned long)fastopen_blob->len);
+    tcp_out_fastopen = fastopen_blob->len > 0 ?  TFO_ATTEMPTED_DATA : TFO_ATTEMPTED_NODATA;
+    }
+
+# elif defined(EXIM_TFO_CONNECTX)
   /* MacOS */
   sa_endpoints_t ends = {
     .sae_srcif = 0, .sae_srcaddr = NULL, .sae_srcaddrlen = 0,
@@ -329,9 +332,9 @@ if (fastopen_blob && f.tcp_fastopen_ok)
 # endif
   }
 else
-#endif	/*TCP_FASTOPEN*/
+#endif	/*EXIM_SUPPORT_TFO*/
   {
-#if defined(TCP_FASTOPEN) && defined(MSG_FASTOPEN)
+#if defined(EXIM_SUPPORT_TFO) && !defined(EXIM_TFO_CONNECTX)
 legacy_connect:
 #endif
 
diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c
index b88fde1b5..c2e234ab5 100644
--- a/src/src/smtp_in.c
+++ b/src/src/smtp_in.c
@@ -2405,20 +2405,35 @@ struct tcp_info tinfo;
 socklen_t len = sizeof(tinfo);
 
 if (getsockopt(fileno(smtp_out), IPPROTO_TCP, TCP_INFO, &tinfo, &len) == 0)
-#ifdef TCPI_OPT_SYN_DATA	/* FreeBSD 11 does not seem to have this yet */
+#  ifdef TCPI_OPT_SYN_DATA	/* FreeBSD 11,12 do not seem to have this yet */
   if (tinfo.tcpi_options & TCPI_OPT_SYN_DATA)
     {
-    DEBUG(D_receive) debug_printf("TCP_FASTOPEN mode connection (ACKd data-on-SYN)\n");
+    DEBUG(D_receive)
+      debug_printf("TCP_FASTOPEN mode connection (ACKd data-on-SYN)\n");
     f.tcp_in_fastopen_data = f.tcp_in_fastopen = TRUE;
     }
   else
-#endif
-    if (tinfo.tcpi_state == TCP_SYN_RECV)
+#  endif
+    if (tinfo.tcpi_state == TCP_SYN_RECV)	/* Not seen on FreeBSD 12.1 */
+    {
+    DEBUG(D_receive)
+      debug_printf("TCP_FASTOPEN mode connection (state TCP_SYN_RECV)\n");
+    f.tcp_in_fastopen = TRUE;
+    }
+#  ifdef __FreeBSD__
+  else if (tinfo.tcpi_options & TCPOPT_FAST_OPEN)
     {
-    DEBUG(D_receive) debug_printf("TCP_FASTOPEN mode connection (state TCP_SYN_RECV)\n");
+    /* This only tells us that some combination of the TCP options was used. It
+    can be a TFO-R received (as of 12.1).  However, pretend it shows real usage
+    (that an acceptable TFO-C was received and acted on).  Ignore the possibility
+    of data-on-SYN for now. */
+    DEBUG(D_receive) debug_printf("TCP_FASTOPEN mode connection (TFO option used)\n");
     f.tcp_in_fastopen = TRUE;
     }
+#  endif
 # endif
+else DEBUG(D_receive)
+  debug_printf("TCP_INFO getsockopt: %s\n", strerror(errno));
 }
 #endif
 
diff --git a/src/src/smtp_out.c b/src/src/smtp_out.c
index 07cc9b762..3dc3a13fb 100644
--- a/src/src/smtp_out.c
+++ b/src/src/smtp_out.c
@@ -155,9 +155,28 @@ return TRUE;
 static void
 tfo_out_check(int sock)
 {
-# if defined(TCP_INFO) && defined(EXIM_HAVE_TCPI_UNACKED)
 struct tcp_info tinfo;
-socklen_t len = sizeof(tinfo);
+int val;
+socklen_t len = sizeof(val);
+
+# ifdef __FreeBSD__
+/* The observability as of 12.1 is not useful as a client, only telling us that
+a TFO option was used on SYN.  It could have been a TFO-R, or ignored by the
+server. */
+
+/*
+if (tcp_out_fastopen == TFO_ATTEMPTED_NODATA || tcp_out_fastopen == TFO_ATTEMPTED_DATA)
+  if (getsockopt(sock, IPPROTO_TCP, TCP_FASTOPEN, &val, &len) == 0 && val != 0) {}
+*/
+switch (tcp_out_fastopen)
+  {
+  case TFO_ATTEMPTED_NODATA:	tcp_out_fastopen = TFO_USED_NODATA; break;
+  case TFO_ATTEMPTED_DATA:	tcp_out_fastopen = TFO_USED_DATA; break;
+  default: break; /* compiler quietening */
+  }
+
+# else	/* Linux & Apple */
+#  if defined(TCP_INFO) && defined(EXIM_HAVE_TCPI_UNACKED)
 
 switch (tcp_out_fastopen)
   {
@@ -205,7 +224,8 @@ switch (tcp_out_fastopen)
 
   default: break; /* compiler quietening */
   }
-# endif
+#  endif
+# endif	/* Linux & Apple */
 }
 #endif
 
diff --git a/src/src/transport.c b/src/src/transport.c
index df7fd1628..14bd91cdb 100644
--- a/src/src/transport.c
+++ b/src/src/transport.c
@@ -172,6 +172,20 @@ for (transport_instance * t = transports; t; t = t->next)
 *             Write block of data                *
 *************************************************/
 
+static int
+tpt_write(int fd, uschar * block, int len, BOOL more, int options)
+{
+return
+#ifndef DISABLE_TLS
+  tls_out.active.sock == fd
+    ? tls_write(tls_out.active.tls_ctx, block, len, more) :
+#endif
+#ifdef MSG_MORE
+  more && !(options & topt_not_socket) ? send(fd, block, len, MSG_MORE) :
+#endif
+  write(fd, block, len);
+}
+
 /* Subroutine called by write_chunk() and at the end of the message actually
 to write a data block. Also called directly by some transports to write
 additional data to the file descriptor (e.g. prefix, suffix).
@@ -215,10 +229,11 @@ Returns:    TRUE on success, FALSE on failure (with errno preserved);
 */
 
 static BOOL
-transport_write_block_fd(transport_ctx * tctx, uschar *block, int len, BOOL more)
+transport_write_block_fd(transport_ctx * tctx, uschar * block, int len, BOOL more)
 {
 int rc, save_errno;
 int local_timeout = transport_write_timeout;
+int connretry = 1;
 int fd = tctx->u.fd;
 
 /* This loop is for handling incomplete writes and other retries. In most
@@ -230,48 +245,42 @@ for (int i = 0; i < 100; i++)
     debug_printf("writing data block fd=%d size=%d timeout=%d%s\n",
       fd, len, local_timeout, more ? " (more expected)" : "");
 
-  /* This code makes use of alarm() in order to implement the timeout. This
-  isn't a very tidy way of doing things. Using non-blocking I/O with select()
-  provides a neater approach. However, I don't know how to do this when TLS is
-  in use. */
-
-  if (transport_write_timeout <= 0)   /* No timeout wanted */
-    {
-    rc =
-#ifndef DISABLE_TLS
-	tls_out.active.sock == fd ? tls_write(tls_out.active.tls_ctx, block, len, more) :
-#endif
-#ifdef MSG_MORE
-	more && !(tctx->options & topt_not_socket)
-	  ? send(fd, block, len, MSG_MORE) :
-#endif
-	write(fd, block, len);
-    save_errno = errno;
-    }
+  /* When doing TCP Fast Open we may get this far before the 3-way handshake
+  is complete, and write returns ENOTCONN.  Detect that, wait for the socket
+  to become writable, and retry once only. */
 
-  /* Timeout wanted. */
-
-  else
+  for(;;)
     {
-    ALARM(local_timeout);
+    fd_set fds;
+    /* This code makes use of alarm() in order to implement the timeout. This
+    isn't a very tidy way of doing things. Using non-blocking I/O with select()
+    provides a neater approach. However, I don't know how to do this when TLS is
+    in use. */
 
-    rc =
-#ifndef DISABLE_TLS
-	tls_out.active.sock == fd ? tls_write(tls_out.active.tls_ctx, block, len, more) :
-#endif
-#ifdef MSG_MORE
-	more && !(tctx->options & topt_not_socket)
-	  ? send(fd, block, len, MSG_MORE) :
-#endif
-	write(fd, block, len);
-
-    save_errno = errno;
-    local_timeout = ALARM_CLR(0);
-    if (sigalrm_seen)
+    if (transport_write_timeout <= 0)   /* No timeout wanted */
       {
-      errno = ETIMEDOUT;
-      return FALSE;
+      rc = tpt_write(fd, block, len, more, tctx->options);
+      save_errno = errno;
       }
+    else				/* Timeout wanted. */
+      {
+      ALARM(local_timeout);
+	rc = tpt_write(fd, block, len, more, tctx->options);
+	save_errno = errno;
+      local_timeout = ALARM_CLR(0);
+      if (sigalrm_seen)
+	{
+	errno = ETIMEDOUT;
+	return FALSE;
+	}
+      }
+
+    if (rc >= 0 || errno != ENOTCONN || connretry <= 0)
+      break;
+
+    FD_ZERO(&fds); FD_SET(fd, &fds);
+    select(fd+1, NULL, &fds, NULL, NULL);	/* could set timout? */
+    connretry--;
     }
 
   /* Hopefully, the most common case is success, so test that first. */