Expansions: A tls option on ${readsocket }. Bug 2282

16 files changed, 199 insertions, 34 deletions

diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt
index d6b65bf8b..8b939b52b 100644
--- a/doc/doc-docbook/spec.xfpt
+++ b/doc/doc-docbook/spec.xfpt
@@ -9878,15 +9878,26 @@ extend what can be done. Firstly, you can vary the timeout. For example:
 .code
 ${readsocket{/socket/name}{request string}{3s}}
 .endd
+
 The third argument is a list of options, of which the first element is the timeout
 and must be present if the argument is given.
 Further elements are options of form &'name=value'&.
-One option type is currently recognised, defining whether (the default)
+Two option types is currently recognised: shutdown and tls.
+The first defines whether (the default)
 or not a shutdown is done on the connection after sending the request.
 Example, to not do so (preferred, eg. by some webservers):
 .code
 ${readsocket{/socket/name}{request string}{3s:shutdown=no}}
 .endd
+.new
+The second, tls, controls the use of TLS on the connection.  Example:
+.code
+${readsocket{/socket/name}{request string}{3s:tls=yes}}
+.endd
+The default is to not use TLS.
+If it is enabled, a shutdown as descripbed above is never done.
+.wen
+
 A fourth argument allows you to change any newlines that are in the data
 that is read, in the same way as for &%readfile%& (see above). This example
 turns them into spaces:
diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog
index 13d8d8236..bc3f8d393 100644
--- a/doc/doc-txt/ChangeLog
+++ b/doc/doc-txt/ChangeLog
@@ -74,6 +74,7 @@ JH/15 Rework TLS client-side context management.  Stop using a global, and
       connection is using TLS; with cutthrough connections this is quite likely.
 
 JH/16 Fix ARC verification to do AS checks in reverse order.
+JH/16 Support a "tls" option on the ${readsocket } expansion item.
 
 
 Exim version 4.91
diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff
index aaf9734f0..7c922cc2e 100644
--- a/doc/doc-txt/NewStuff
+++ b/doc/doc-txt/NewStuff
@@ -14,6 +14,9 @@ Version 4.92
     when individual headers are wrapped onto multiple lines; with previous
     facilities hard to parse.
 
+ 2. The ${readsocket } expansion item now takes a "tls" option, doing the
+    obvious thing.
+
 Version 4.91
 --------------
 
diff --git a/src/src/expand.c b/src/src/expand.c
index b9eeb7c46..596fb2404 100644
--- a/src/src/expand.c
+++ b/src/src/expand.c
@@ -3550,6 +3550,26 @@ return yield;
 }
 
 
+#ifdef SUPPORT_TLS
+static gstring *
+cat_file_tls(void * tls_ctx, gstring * yield, uschar * eol)
+{
+int rc;
+uschar * s;
+uschar buffer[1024];
+
+while ((rc = tls_read(tls_ctx, buffer, sizeof(buffer))) > 0)
+  for (s = buffer; rc--; s++)
+    yield = eol && *s == '\n'
+      ? string_cat(yield, eol) : string_catn(yield, s, 1);
+
+/* We assume that all errors, and any returns of zero bytes,
+are actually EOF. */
+
+(void) string_from_gstring(yield);
+return yield;
+}
+#endif
 
 
 /*************************************************
@@ -4801,9 +4821,15 @@ while (*s != 0)
       int timeout = 5;
       int save_ptr = yield->ptr;
       FILE *f;
-      uschar *arg;
-      uschar *sub_arg[4];
+      uschar * arg;
+      uschar * sub_arg[4];
+      uschar * server_name = NULL;
+      host_item host;
       BOOL do_shutdown = TRUE;
+#ifdef SUPPORT_TLS
+      BOOL do_tls = FALSE;
+      void * tls_ctx = NULL;
+#endif
       blob reqstr;
 
       if (expand_forbid & RDO_READSOCK)
@@ -4846,10 +4872,14 @@ while (*s != 0)
 
 	while ((item = string_nextinlist(&list, &sep, NULL, 0)))
 	  if (Ustrncmp(item, US"shutdown=", 9) == 0)
-	    if (Ustrcmp(item + 9, US"no") == 0)
-	      do_shutdown = FALSE;
+	    { if (Ustrcmp(item + 9, US"no") == 0) do_shutdown = FALSE; }
+#ifdef SUPPORT_TLS
+	  else if (Ustrncmp(item, US"tls=", 4) == 0)
+	    { if (Ustrcmp(item + 9, US"no") != 0) do_tls = TRUE; }
+#endif
         }
-      else sub_arg[3] = NULL;                     /* No eol if no timeout */
+      else
+	sub_arg[3] = NULL;                     /* No eol if no timeout */
 
       /* If skipping, we don't actually do anything. Otherwise, arrange to
       connect to either an IP or a Unix socket. */
@@ -4861,8 +4891,10 @@ while (*s != 0)
         if (Ustrncmp(sub_arg[0], "inet:", 5) == 0)
           {
           int port;
-          uschar * server_name = sub_arg[0] + 5;
-          uschar * port_name = Ustrrchr(server_name, ':');
+          uschar * port_name;
+
+          server_name = sub_arg[0] + 5;
+          port_name = Ustrrchr(server_name, ':');
 
           /* Sort out the port */
 
@@ -4898,11 +4930,12 @@ while (*s != 0)
             }
 
 	  fd = ip_connectedsocket(SOCK_STREAM, server_name, port, port,
-		  timeout, NULL, &expand_string_message, &reqstr);
+		  timeout, &host, &expand_string_message,
+		  do_tls ? NULL : &reqstr);
 	  callout_address = NULL;
 	  if (fd < 0)
               goto SOCK_FAIL;
-	  reqstr.len = 0;
+	  if (!do_tls) reqstr.len = 0;
           }
 
         /* Handle a Unix domain socket */
@@ -4922,6 +4955,7 @@ while (*s != 0)
           sockun.sun_family = AF_UNIX;
           sprintf(sockun.sun_path, "%.*s", (int)(sizeof(sockun.sun_path)-1),
             sub_arg[0]);
+	  server_name = sockun.sun_path;
 
           sigalrm_seen = FALSE;
           alarm(timeout);
@@ -4938,10 +4972,27 @@ while (*s != 0)
               "%s: %s", sub_arg[0], strerror(errno));
             goto SOCK_FAIL;
             }
+	  host.name = server_name;
+	  host.address = US"";
           }
 
         DEBUG(D_expand) debug_printf_indent("connected to socket %s\n", sub_arg[0]);
 
+#ifdef SUPPORT_TLS
+	if (do_tls)
+	  {
+	  tls_support tls_dummy = {0};
+	  uschar * errstr;
+
+	  if (!(tls_ctx = tls_client_start(fd, &host, NULL, NULL, NULL,
+	  			&tls_dummy, &errstr)))
+	    {
+	    expand_string_message = string_sprintf("TLS connect failed: %s", errstr);
+	    goto SOCK_FAIL;
+	    }
+	  }
+#endif
+
 	/* Allow sequencing of test actions */
 	if (running_in_test_harness) millisleep(100);
 
@@ -4951,7 +5002,11 @@ while (*s != 0)
           {
           DEBUG(D_expand) debug_printf_indent("writing \"%s\" to socket\n",
             reqstr.data);
-          if (write(fd, reqstr.data, reqstr.len) != reqstr.len)
+          if ( (
+#ifdef SUPPORT_TLS
+	      tls_ctx ? tls_write(tls_ctx, reqstr.data, reqstr.len, FALSE) :
+#endif
+			write(fd, reqstr.data, reqstr.len)) != reqstr.len)
             {
             expand_string_message = string_sprintf("request write to socket "
               "failed: %s", strerror(errno));
@@ -4964,7 +5019,7 @@ while (*s != 0)
         system doesn't have this function, make it conditional. */
 
 #ifdef SHUT_WR
-	if (do_shutdown) shutdown(fd, SHUT_WR);
+	if (!tls_ctx && do_shutdown) shutdown(fd, SHUT_WR);
 #endif
 
 	if (running_in_test_harness) millisleep(100);
@@ -4972,12 +5027,26 @@ while (*s != 0)
         /* Now we need to read from the socket, under a timeout. The function
         that reads a file can be used. */
 
-        f = fdopen(fd, "rb");
+	if (!tls_ctx)
+	  f = fdopen(fd, "rb");
         sigalrm_seen = FALSE;
         alarm(timeout);
-        yield = cat_file(f, yield, sub_arg[3]);
+        yield =
+#ifdef SUPPORT_TLS
+	  tls_ctx ? cat_file_tls(tls_ctx, yield, sub_arg[3]) :
+#endif
+		    cat_file(f, yield, sub_arg[3]);
         alarm(0);
-        (void)fclose(f);
+
+#ifdef SUPPORT_TLS
+	if (tls_ctx)
+	  {
+	  tls_close(tls_ctx, TRUE);
+	  close(fd);
+	  }
+	else
+#endif
+	  (void)fclose(f);
 
         /* After a timeout, we restore the pointer in the result, that is,
         make sure we add nothing from the socket. */
diff --git a/src/src/ip.c b/src/src/ip.c
index 555dc2d84..82876c62e 100644
--- a/src/src/ip.c
+++ b/src/src/ip.c
@@ -262,6 +262,7 @@ if (fastopen_blob && tcp_fastopen_ok)
     DEBUG(D_transport|D_v)
       debug_printf("non-TFO mode connection attempt to %s, %lu data\n",
 	address, (unsigned long)fastopen_blob->len);
+    /*XXX also seen on successful TFO, sigh */
     tcp_out_fastopen = fastopen_blob->len > 0 ?  2 : 1;
     }
   else if (errno == EINPROGRESS)	/* expected if we had no cookie for peer */
@@ -339,7 +340,7 @@ return -1;
 Arguments:
   type          SOCK_DGRAM or SOCK_STREAM
   af            AF_INET6 or AF_INET for the socket type
-  address       the remote address, in text form
+  hostname	host name, or ip address (as text)
   portlo,porthi the remote port range
   timeout       a timeout
   connhost	if not NULL, host_item to be filled in with connection details
diff --git a/src/src/tls-gnu.c b/src/src/tls-gnu.c
index 12c9fdb38..dfe09200b 100644
--- a/src/src/tls-gnu.c
+++ b/src/src/tls-gnu.c
@@ -125,7 +125,7 @@ typedef struct exim_gnutls_state {
   BOOL			trigger_sni_changes;
   BOOL			have_set_peerdn;
   const struct host_item *host;
-  gnutls_x509_crt_t 	peercert;
+  gnutls_x509_crt_t	peercert;
   uschar		*peerdn;
   uschar		*ciphersuite;
   uschar		*received_sni;
@@ -2241,7 +2241,7 @@ return TRUE;
 
 Arguments:
   fd                the fd of the connection
-  host              connected host (for messages)
+  host              connected host (for messages and option-tests)
   addr              the first address (not used)
   tb                transport (always smtp)
   tlsa_dnsa	    non-NULL, either request or require dane for this host, and
@@ -2264,8 +2264,9 @@ tls_client_start(int fd, host_item *host,
 #endif
     tls_support * tlsp, uschar ** errstr)
 {
-smtp_transport_options_block *ob =
-  (smtp_transport_options_block *)tb->options_block;
+smtp_transport_options_block *ob = tb
+  ? (smtp_transport_options_block *)tb->options_block
+  : &smtp_transport_option_defaults;
 int rc;
 exim_gnutls_state_st * state = NULL;
 uschar *cipher_list = NULL;
@@ -2375,7 +2376,7 @@ if (request_ocsp)
 #endif
 
 #ifndef DISABLE_EVENT
-if (tb->event_action)
+if (tb && tb->event_action)
   {
   state->event_action = tb->event_action;
   gnutls_session_set_ptr(state->session, state);
@@ -2477,7 +2478,7 @@ would tamper with the TLS session in the parent process).
 Arguments:
   ct_ctx	client context pointer, or NULL for the one global server context
   shutdown	1 if TLS close-alert is to be sent,
- 		2 if also response to be waited for
+		2 if also response to be waited for
 
 Returns:     nothing
 */
@@ -2678,7 +2679,7 @@ Arguments:
   len       size of buffer
 
 Returns:    the number of bytes read
-            -1 after a failed read
+            -1 after a failed read, including EOF
 */
 
 int
diff --git a/src/src/tls-openssl.c b/src/src/tls-openssl.c
index adabc963e..d8c8101cc 100644
--- a/src/src/tls-openssl.c
+++ b/src/src/tls-openssl.c
@@ -436,7 +436,7 @@ else
 
   if (  tlsp == &tls_out
      && ((verify_cert_hostnames = client_static_cbinfo->verify_cert_hostnames)))
-     	/* client, wanting hostname check */
+	/* client, wanting hostname check */
     {
 
 #ifdef EXIM_HAVE_OPENSSL_CHECKHOST
@@ -1094,7 +1094,7 @@ if (!cbinfo->certificate)
   {
   if (!cbinfo->is_server)		/* client */
     return OK;
-  					/* server */
+					/* server */
   if (tls_install_selfsign(sctx, errstr) != OK)
     return DEFER;
   }
@@ -2032,14 +2032,14 @@ server_verify_callback_called = FALSE;
 if (verify_check_host(&tls_verify_hosts) == OK)
   {
   rc = setup_certs(server_ctx, tls_verify_certificates, tls_crl, NULL,
-  			FALSE, verify_callback_server, errstr);
+			FALSE, verify_callback_server, errstr);
   if (rc != OK) return rc;
   server_verify_optional = FALSE;
   }
 else if (verify_check_host(&tls_try_verify_hosts) == OK)
   {
   rc = setup_certs(server_ctx, tls_verify_certificates, tls_crl, NULL,
-  			TRUE, verify_callback_server, errstr);
+			TRUE, verify_callback_server, errstr);
   if (rc != OK) return rc;
   server_verify_optional = TRUE;
   }
@@ -2251,11 +2251,11 @@ return DEFER;
 
 Argument:
   fd               the fd of the connection
-  host             connected host (for messages)
-  addr             the first address
+  host             connected host (for messages and option-tests)
+  addr             the first address (for some randomness; can be NULL)
   tb               transport (always smtp)
   tlsa_dnsa        tlsa lookup, if DANE, else null
-  tlsp		   record details of channel configuration
+  tlsp		   record details of channel configuration here; must be non-NULL
   errstr	   error string pointer
 
 Returns:           Pointer to TLS session context, or NULL on error
@@ -2269,8 +2269,9 @@ tls_client_start(int fd, host_item *host, address_item *addr,
 #endif
   tls_support * tlsp, uschar ** errstr)
 {
-smtp_transport_options_block * ob =
-  (smtp_transport_options_block *)tb->options_block;
+smtp_transport_options_block * ob = tb
+  ? (smtp_transport_options_block *)tb->options_block
+  : &smtp_transport_option_defaults;
 exim_openssl_client_tls_ctx * exim_client_ctx;
 static uschar peerdn[256];
 uschar * expciphers;
@@ -2457,7 +2458,7 @@ if (request_ocsp)
 #endif
 
 #ifndef DISABLE_EVENT
-client_static_cbinfo->event_action = tb->event_action;
+client_static_cbinfo->event_action = tb ? tb->event_action : NULL;
 #endif
 
 /* There doesn't seem to be a built-in timeout on connection. */
@@ -2666,7 +2667,7 @@ Arguments:
   len       size of buffer
 
 Returns:    the number of bytes read
-            -1 after a failed read
+            -1 after a failed read, including EOF
 
 Only used by the client-side TLS.
 */
diff --git a/test/confs/2099 b/test/confs/2099
new file mode 100644
index 000000000..ce24e0967
--- /dev/null
+++ b/test/confs/2099
@@ -0,0 +1,16 @@
+# Exim test configuration 2019
+
+.include DIR/aux-var/tls_conf_prefix
+
+primary_hostname = myhost.test.ex
+
+# ----- Main settings -----
+
+log_selector = +tls_peerdn
+
+tls_advertise_hosts = *
+
+tls_certificate = DIR/aux-fixed/cert1
+tls_privatekey = DIR/aux-fixed/cert1
+
+# End
diff --git a/test/confs/2199 b/test/confs/2199
new file mode 100644
index 000000000..db3fbb0f0
--- /dev/null
+++ b/test/confs/2199
@@ -0,0 +1,16 @@
+# Exim test configuration 2119
+
+.include DIR/aux-var/tls_conf_prefix
+
+primary_hostname = myhost.test.ex
+
+# ----- Main settings -----
+
+log_selector = +tls_peerdn
+
+tls_advertise_hosts = *
+
+tls_certificate = DIR/aux-fixed/cert1
+tls_privatekey = DIR/aux-fixed/cert1
+
+# End
diff --git a/test/log/2099 b/test/log/2099
new file mode 100644
index 000000000..eb9d772d0
--- /dev/null
+++ b/test/log/2099
@@ -0,0 +1,3 @@
+
+******** SERVER ********
+1999-03-02 09:44:33 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTPS on port 1225
diff --git a/test/log/2199 b/test/log/2199
new file mode 100644
index 000000000..eb9d772d0
--- /dev/null
+++ b/test/log/2199
@@ -0,0 +1,3 @@
+
+******** SERVER ********
+1999-03-02 09:44:33 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTPS on port 1225
diff --git a/test/scripts/2000-GnuTLS/2099 b/test/scripts/2000-GnuTLS/2099
new file mode 100644
index 000000000..632dc094b
--- /dev/null
+++ b/test/scripts/2000-GnuTLS/2099
@@ -0,0 +1,14 @@
+# ${readsocket (IPv4 TLS)
+need_ipv4
+#
+exim -DSERVER=server -tls-on-connect -bd -oX PORT_D
+****
+#
+#
+millisleep 500
+exim -be
+1 >>${readsocket{inet:thisloop:PORT_D}{QUIT\n}{2s:tls=yes}}<<
+****
+millisleep 500
+#
+killdaemon
diff --git a/test/scripts/2100-OpenSSL/2199 b/test/scripts/2100-OpenSSL/2199
new file mode 100644
index 000000000..632dc094b
--- /dev/null
+++ b/test/scripts/2100-OpenSSL/2199
@@ -0,0 +1,14 @@
+# ${readsocket (IPv4 TLS)
+need_ipv4
+#
+exim -DSERVER=server -tls-on-connect -bd -oX PORT_D
+****
+#
+#
+millisleep 500
+exim -be
+1 >>${readsocket{inet:thisloop:PORT_D}{QUIT\n}{2s:tls=yes}}<<
+****
+millisleep 500
+#
+killdaemon
diff --git a/test/stderr/2199 b/test/stderr/2199
new file mode 100644
index 000000000..0423be152
--- /dev/null
+++ b/test/stderr/2199
@@ -0,0 +1,4 @@
+1999-03-02 09:44:33 [NULL] SSL verify error: depth=0 error=self signed certificate cert=/C=UK/O=The Exim Maintainers/OU=Test Suite/CN=Phil Pennock
+1999-03-02 09:44:33 [NULL] SSL verify error: certificate name mismatch: DN="/C=UK/O=The Exim Maintainers/OU=Test Suite/CN=Phil Pennock" H="thisloop"
+
+******** SERVER ********
diff --git a/test/stdout/2099 b/test/stdout/2099
new file mode 100644
index 000000000..a3eab5117
--- /dev/null
+++ b/test/stdout/2099
@@ -0,0 +1,4 @@
+> 1 >>220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000

+221 myhost.test.ex closing connection

+<<
+> 
diff --git a/test/stdout/2199 b/test/stdout/2199
new file mode 100644
index 000000000..a3eab5117
--- /dev/null
+++ b/test/stdout/2199
@@ -0,0 +1,4 @@
+> 1 >>220 myhost.test.ex ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000

+221 myhost.test.ex closing connection

+<<
+>