TFO: early-data for ClamAV and for readsocket expansion

7 files changed, 75 insertions, 46 deletions

diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff
index d975fe14a..88def2629 100644
--- a/doc/doc-txt/NewStuff
+++ b/doc/doc-txt/NewStuff
@@ -49,8 +49,7 @@ Version 4.90
 
 12. TCP Fast Open logging.  As a server, logs when the SMTP banner was sent
     while still in SYN_RECV state; as a client logs when the connection
-    is opened with a TFO cookie.  Support varies between platforms
-    (Linux does both. FreeBSD server only, others unknown).
+    is opened with a TFO cookie.
 
 13. DKIM support for multiple signing, by domain and/or key-selector.
     DKIM support for multiple hashes.
@@ -58,6 +57,9 @@ Version 4.90
 14. Exipick understands -C|--config for an alternative Exim
     configuration file.
 
+15. TCP Fast Open used, with data-on-SYN, for client SMTP via SOCKS5 proxy,
+    for ${readsocket } expansions, and for ClamAV.
+
 
 Version 4.89
 ------------
diff --git a/src/src/acl.c b/src/src/acl.c
index 619f6f287..b5ffa0193 100644
--- a/src/src/acl.c
+++ b/src/src/acl.c
@@ -2771,8 +2771,9 @@ if (r == HOST_FIND_FAILED || r == HOST_FIND_AGAIN)
 HDEBUG(D_acl)
   debug_printf_indent("udpsend [%s]:%d %s\n", h->address, portnum, arg);
 
+/*XXX this could better use sendto */
 r = s = ip_connectedsocket(SOCK_DGRAM, h->address, portnum, portnum,
-		1, NULL, &errstr);
+		1, NULL, &errstr, NULL);
 if (r < 0) goto defer;
 len = Ustrlen(arg);
 r = send(s, arg, len, 0);
diff --git a/src/src/expand.c b/src/src/expand.c
index 04bb92916..353b8ea56 100644
--- a/src/src/expand.c
+++ b/src/src/expand.c
@@ -4708,8 +4708,7 @@ while (*s != 0)
 
       /* Open the file and read it */
 
-      f = Ufopen(sub_arg[0], "rb");
-      if (f == NULL)
+      if (!(f = Ufopen(sub_arg[0], "rb")))
         {
         expand_string_message = string_open_failed(errno, "%s", sub_arg[0]);
         goto EXPAND_FAILED;
@@ -4720,7 +4719,8 @@ while (*s != 0)
       continue;
       }
 
-    /* Handle "readsocket" to insert data from a Unix domain socket */
+    /* Handle "readsocket" to insert data from a socket, either
+    Inet or Unix domain */
 
     case EITEM_READSOCK:
       {
@@ -4728,10 +4728,10 @@ while (*s != 0)
       int timeout = 5;
       int save_ptr = ptr;
       FILE *f;
-      struct sockaddr_un sockun;         /* don't call this "sun" ! */
       uschar *arg;
       uschar *sub_arg[4];
       BOOL do_shutdown = TRUE;
+      blob reqstr;
 
       if (expand_forbid & RDO_READSOCK)
         {
@@ -4749,6 +4749,11 @@ while (*s != 0)
         case 3: goto EXPAND_FAILED;
         }
 
+      /* Grab the request string, if any */
+
+      reqstr.data = sub_arg[1];
+      reqstr.len = Ustrlen(sub_arg[1]);
+
       /* Sort out timeout, if given.  The second arg is a list with the first element
       being a time value.  Any more are options of form "name=value".  Currently the
       only option recognised is "shutdown". */
@@ -4783,12 +4788,12 @@ while (*s != 0)
         if (Ustrncmp(sub_arg[0], "inet:", 5) == 0)
           {
           int port;
-          uschar *server_name = sub_arg[0] + 5;
-          uschar *port_name = Ustrrchr(server_name, ':');
+          uschar * server_name = sub_arg[0] + 5;
+          uschar * port_name = Ustrrchr(server_name, ':');
 
           /* Sort out the port */
 
-          if (port_name == NULL)
+          if (!port_name)
             {
             expand_string_message =
               string_sprintf("missing port for readsocket %s", sub_arg[0]);
@@ -4810,7 +4815,7 @@ while (*s != 0)
           else
             {
             struct servent *service_info = getservbyname(CS port_name, "tcp");
-            if (service_info == NULL)
+            if (!service_info)
               {
               expand_string_message = string_sprintf("unknown port \"%s\"",
                 port_name);
@@ -4820,17 +4825,20 @@ while (*s != 0)
             }
 
 	  fd = ip_connectedsocket(SOCK_STREAM, server_name, port, port,
-		  timeout, NULL, &expand_string_message);
+		  timeout, NULL, &expand_string_message, &reqstr);
 	  callout_address = NULL;
 	  if (fd < 0)
               goto SOCK_FAIL;
+	  reqstr.len = 0;
           }
 
         /* Handle a Unix domain socket */
 
         else
           {
+	  struct sockaddr_un sockun;         /* don't call this "sun" ! */
           int rc;
+
           if ((fd = socket(PF_UNIX, SOCK_STREAM, 0)) == -1)
             {
             expand_string_message = string_sprintf("failed to create socket: %s",
@@ -4864,14 +4872,13 @@ while (*s != 0)
 	/* Allow sequencing of test actions */
 	if (running_in_test_harness) millisleep(100);
 
-        /* Write the request string, if not empty */
+        /* Write the request string, if not empty or already done */
 
-        if (sub_arg[1][0] != 0)
+        if (reqstr.len)
           {
-          int len = Ustrlen(sub_arg[1]);
           DEBUG(D_expand) debug_printf_indent("writing \"%s\" to socket\n",
-            sub_arg[1]);
-          if (write(fd, sub_arg[1], len) != len)
+            reqstr.data);
+          if (write(fd, reqstr.data, reqstr.len) != reqstr.len)
             {
             expand_string_message = string_sprintf("request write to socket "
               "failed: %s", strerror(errno));
diff --git a/src/src/functions.h b/src/src/functions.h
index a96ffb69c..9d1f6dc6e 100644
--- a/src/src/functions.h
+++ b/src/src/functions.h
@@ -237,7 +237,7 @@ extern int     ip_addr(void *, int, const uschar *, int);
 extern int     ip_bind(int, int, uschar *, int);
 extern int     ip_connect(int, int, const uschar *, int, int, const blob *);
 extern int     ip_connectedsocket(int, const uschar *, int, int,
-                 int, host_item *, uschar **);
+                 int, host_item *, uschar **, const blob *);
 extern int     ip_get_address_family(int);
 extern void    ip_keepalive(int, const uschar *, BOOL);
 extern int     ip_recv(int, uschar *, int, int);
diff --git a/src/src/ip.c b/src/src/ip.c
index 258ab5c23..266eaf414 100644
--- a/src/src/ip.c
+++ b/src/src/ip.c
@@ -312,23 +312,22 @@ Arguments:
   address       the remote address, in text form
   portlo,porthi the remote port range
   timeout       a timeout
-  connhost	if not NULL, host_item filled in with connection details
+  connhost	if not NULL, host_item to be filled in with connection details
   errstr        pointer for allocated string on error
-XXX could add early-data support
+  fastopen	with SOCK_STREAM, if non-null, request TCP Fast Open.
+		Additionally, optional early-data to send
 
 Return:
   socket fd, or -1 on failure (having allocated an error string)
 */
 int
 ip_connectedsocket(int type, const uschar * hostname, int portlo, int porthi,
-	int timeout, host_item * connhost, uschar ** errstr)
+      int timeout, host_item * connhost, uschar ** errstr, const blob * fastopen)
 {
 int namelen, port;
 host_item shost;
 host_item *h;
 int af = 0, fd, fd4 = -1, fd6 = -1;
-blob * fastopen = tcp_fastopen_ok && type == SOCK_STREAM
-  ? &tcp_fastopen_nodata : NULL;
 
 shost.next = NULL;
 shost.address = NULL;
@@ -406,6 +405,7 @@ bad:
 }
 
 
+/*XXX TFO? */
 int
 ip_tcpsocket(const uschar * hostport, uschar ** errstr, int tmo)
 {
@@ -426,7 +426,7 @@ if (scan != 3)
   }
 
 return ip_connectedsocket(SOCK_STREAM, hostname, portlow, porthigh,
-			  tmo, NULL, errstr);
+			  tmo, NULL, errstr, NULL);
 }
 
 int
diff --git a/src/src/malware.c b/src/src/malware.c
index b626b18a8..32f2e9e49 100644
--- a/src/src/malware.c
+++ b/src/src/malware.c
@@ -147,9 +147,10 @@ uses the returned in_addr to get a second connection to the same system.
 */
 static inline int
 m_tcpsocket(const uschar * hostname, unsigned int port,
-	host_item * host, uschar ** errstr)
+	host_item * host, uschar ** errstr, const blob * fastopen)
 {
-return ip_connectedsocket(SOCK_STREAM, hostname, port, port, 5, host, errstr);
+return ip_connectedsocket(SOCK_STREAM, hostname, port, port, 5,
+			  host, errstr, fastopen);
 }
 
 static int
@@ -1254,6 +1255,7 @@ badseek:  err = errno;
 #else
       uint32_t send_size, send_final_zeroblock;
 #endif
+      blob cmd_str;
 
       /*XXX if unixdomain socket, only one server supported. Needs fixing;
       there's no reason we should not mix local and remote servers */
@@ -1349,6 +1351,19 @@ badseek:  err = errno;
 	  string_sprintf("local/SCAN mode incompatible with" \
 	    " : in path to email filename [%s]", eml_filename));
 
+      /* Set up the very first data we will be sending */
+      if (!use_scan_command)
+#ifdef WITH_OLD_CLAMAV_STREAM
+	{ cmd_str.data = US"STREAM\n"; cmd_str.len = 7; }
+#else
+	{ cmd_str.data = US"zINSTREAM"; cmd_str.len = 10; }
+#endif
+      else
+	{
+	cmd_str.data = string_sprintf("SCAN %s\n", eml_filename);
+	cmd_str.len = Ustrlen(cmd_str.data);
+	}
+
       /* We have some network servers specified */
       if (num_servers)
 	{
@@ -1358,7 +1373,7 @@ badseek:  err = errno;
 
 	while (num_servers > 0)
 	  {
-	  int i = random_number( num_servers );
+	  int i = random_number(num_servers);
 	  clamd_address * cd = cv[i];
 
 	  DEBUG(D_acl) debug_printf_indent("trying server name %s, port %u\n",
@@ -1368,11 +1383,12 @@ badseek:  err = errno;
 	   * on both connections (as one host could resolve to multiple ips) */
 	  for (;;)
 	    {
-	    sock= m_tcpsocket(cd->hostspec, cd->tcp_port, &connhost, &errstr);
-	    if (sock >= 0)
+	    if ((sock = m_tcpsocket(cd->hostspec, cd->tcp_port,
+				    &connhost, &errstr, &cmd_str)) >= 0)
 	      {
 	      /* Connection successfully established with a server */
 	      hostname = cd->hostspec;
+	      cmd_str.len = 0;
 	      break;
 	      }
 	    if (cd->retry <= 0) break;
@@ -1421,9 +1437,10 @@ badseek:  err = errno;
 	    "Malware scan: issuing %s old-style remote scan (PORT)\n",
 	    scanner_name);
 
-	/* Pass the string to ClamAV (7 = "STREAM\n") */
-	if (m_sock_send(sock, US"STREAM\n", 7, &errstr) < 0)
-	  return m_errlog_defer(scanent, CUS callout_address, errstr);
+	/* Pass the string to ClamAV (7 = "STREAM\n"), if not already sent */
+	if (cmd_str.len)
+	  if (m_sock_send(sock, cmd_str.data, cmd_str.len, &errstr) < 0)
+	    return m_errlog_defer(scanent, CUS callout_address, errstr);
 
 	memset(av_buffer2, 0, sizeof(av_buffer2));
 	bread = ip_recv(sock, av_buffer2, sizeof(av_buffer2), tmo-time(NULL));
@@ -1443,13 +1460,13 @@ badseek:  err = errno;
 		  "ClamAV returned null", sock);
 
 	av_buffer2[bread] = '\0';
-	if( sscanf(CS av_buffer2, "PORT %u\n", &port) != 1 )
+	if(sscanf(CS av_buffer2, "PORT %u\n", &port) != 1)
 	  return m_errlog_defer_3(scanent, CUS callout_address,
 	    string_sprintf("Expected port information from clamd, got '%s'",
 	      av_buffer2),
 	    sock);
 
-	sockData = m_tcpsocket(connhost.address, port, NULL, &errstr);
+	sockData = m_tcpsocket(connhost.address, port, NULL, &errstr, NULL);
 	if (sockData < 0)
 	  return m_errlog_defer_3(scanent, CUS callout_address, errstr, sock);
 
@@ -1463,12 +1480,13 @@ badseek:  err = errno;
 	    "Malware scan: issuing %s new-style remote scan (zINSTREAM)\n",
 	    scanner_name);
 
-	/* Pass the string to ClamAV (10 = "zINSTREAM\0") */
-	if (send(sock, "zINSTREAM", 10, 0) < 0)
-	  return m_errlog_defer_3(scanent, CUS hostname,
-	    string_sprintf("unable to send zINSTREAM to socket (%s)",
-	      strerror(errno)),
-	    sock);
+	/* Pass the string to ClamAV (10 = "zINSTREAM\0"), if not already sent */
+	if (cmd_str.len)
+	  if (send(sock, cmd_str.data, cmd_str.len, 0) < 0)
+	    return m_errlog_defer_3(scanent, CUS hostname,
+	      string_sprintf("unable to send zINSTREAM to socket (%s)",
+		strerror(errno)),
+	      sock);
 
 # define CLOSE_SOCKDATA /**/
 #endif
@@ -1569,17 +1587,17 @@ b_seek:   err = errno;
 	scanned twice, in the broken out files and from the original .eml.
 	Since ClamAV now handles emails (and has for quite some time) we can
 	just use the email file itself. */
-	/* Pass the string to ClamAV (7 = "SCAN \n" + \0) */
-	file_name = string_sprintf("SCAN %s\n", eml_filename);
+	/* Pass the string to ClamAV (7 = "SCAN \n" + \0), if not already sent */
 
 	DEBUG(D_acl) debug_printf_indent(
 	    "Malware scan: issuing %s local-path scan [%s]\n",
 	    scanner_name, scanner_options);
 
-	if (send(sock, file_name, Ustrlen(file_name), 0) < 0)
-	  return m_errlog_defer_3(scanent, CUS callout_address,
-	    string_sprintf("unable to write to socket (%s)", strerror(errno)),
-	    sock);
+	if (cmd_str.len)
+	  if (send(sock, cmd_str.data, cmd_str.len, 0) < 0)
+	    return m_errlog_defer_3(scanent, CUS callout_address,
+	      string_sprintf("unable to write to socket (%s)", strerror(errno)),
+	      sock);
 
 	/* Do not shut down the socket for writing; a user report noted that
 	 * clamd 0.70 does not react well to this. */
diff --git a/src/src/spam.c b/src/src/spam.c
index fa3a4fb68..20154da4f 100644
--- a/src/src/spam.c
+++ b/src/src/spam.c
@@ -343,6 +343,7 @@ start = time(NULL);
 
     for (;;)
       {
+      /*XXX could potentially use TFO early-data here */
       if (  (spamd_sock = ip_streamsocket(sd->hostspec, &errstr, 5)) >= 0
          || sd->retry <= 0
 	 )