Add tls_verify_hosts and tls_try_verify_hosts to smtp transport. Bug 1371

Code by Wolfgang Breyha, docs and testsuite by Jeremy Harris

12 files changed, 170 insertions, 15 deletions

diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt
index cab1c82dd..ae4d75ecb 100644
--- a/doc/doc-docbook/spec.xfpt
+++ b/doc/doc-docbook/spec.xfpt
@@ -23027,6 +23027,14 @@ unknown state), opens a new one to the same host, and then tries the delivery
 in clear.
 
 
+.option tls_try_verify_hosts smtp "host list&!! unset
+.cindex "TLS" "server certificate verification"
+.cindex "certificate" "verification of server"
+For OpenSSL only, this option gives a list of hosts for which, on encrypted connections,
+certificate verification will be tried but need not succeed.
+The &%tls_verify_certificates%& option must also be set.
+
+
 .option tls_verify_certificates smtp string&!! unset
 .cindex "TLS" "server certificate verification"
 .cindex "certificate" "verification of server"
@@ -23041,6 +23049,20 @@ single file if you are using GnuTLS. The values of &$host$& and
 &$host_address$& are set to the name and address of the server during the
 expansion of this option. See chapter &<<CHAPTLS>>& for details of TLS.
 
+For back-compatability, or when GnuTLS is used,
+if neither tls_verify_hosts nor tls_try_verify_hosts are set
+and certificate verification fails the TLS connection is closed.
+
+
+.option tls_verify_hosts smtp "host list&!! unset
+.cindex "TLS" "server certificate verification"
+.cindex "certificate" "verification of server"
+For OpenSSL only, this option gives a list of hosts for which. on encrypted connections,
+certificate verification must succeed.
+The &%tls_verify_certificates%& option must also be set.
+If both this option and &%tls_try_verify_hosts%& are unset
+operation is as if this option selected all hosts.
+
 
 
 
@@ -25942,6 +25964,12 @@ for OpenSSL only (not GnuTLS), a directory, that contains a collection of
 expected server certificates. The client verifies the server's certificate
 against this collection, taking into account any revoked certificates that are
 in the list defined by &%tls_crl%&.
+Failure to verify fails the TLS connection unless either of the
+&%tls_verify_hosts%& or &%tls_try_verify_hosts%& options are set.
+
+The &%tls_verify_hosts%& and &%tls_try_verify_hosts%& options restrict
+certificate verification to the listed servers.  Verification either must
+or need not succeed respectively.
 
 If
 &%tls_require_ciphers%& is set on the &(smtp)& transport, it must contain a
diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog
index 04a7ce02e..c29f21cbf 100644
--- a/doc/doc-txt/ChangeLog
+++ b/doc/doc-txt/ChangeLog
@@ -56,6 +56,8 @@ JH/06 Log outbound-TLS and port details, subject to log selectors, for a
 
 JH/07 Add malware type "sock" for talking to simple daemon.
 
+JH/08 Bugzilla 1371: Add tls_{,try_}verify_hosts to smtp transport. OpenSSL only.
+
 
 Exim version 4.82
 -----------------
diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff
index c4de902c0..95b4119d1 100644
--- a/doc/doc-txt/NewStuff
+++ b/doc/doc-txt/NewStuff
@@ -27,6 +27,11 @@ Version 4.83
     and a second regex to extract malware_name.  The mail spoofile name can
     be included in the command line.
 
+ 5. When built with OpenSSL the smtp transport now supports options
+    "tls_verify_hosts" and "tls_try_verify_hosts".  If either is set the
+    certificate verification is split from the encryption operation. The
+    default remains that a failed verification cancels the encryption.
+
 
 Version 4.82
 ------------
diff --git a/src/src/functions.h b/src/src/functions.h
index a94fac3f2..e92c2455f 100644
--- a/src/src/functions.h
+++ b/src/src/functions.h
@@ -30,7 +30,7 @@ extern int     tls_client_start(int, host_item *, address_item *,
 # ifdef EXPERIMENTAL_OCSP
                  uschar *,
 # endif
-                 int, int);
+                 int, int, uschar *, uschar *);
 extern void    tls_close(BOOL, BOOL);
 extern int     tls_feof(void);
 extern int     tls_ferror(void);
diff --git a/src/src/tls-gnu.c b/src/src/tls-gnu.c
index 4f1169aa4..280744ec0 100644
--- a/src/src/tls-gnu.c
+++ b/src/src/tls-gnu.c
@@ -1563,6 +1563,8 @@ Arguments:
   require_ciphers   list of allowed ciphers or NULL
   dh_min_bits       minimum number of bits acceptable in server's DH prime
   timeout           startup timeout
+  verify_hosts     mandatory client verification 
+  try_verify_hosts optional client verification
 
 Returns:            OK/DEFER/FAIL (because using common functions),
                     but for a client, DEFER and FAIL have the same meaning
@@ -1577,7 +1579,8 @@ tls_client_start(int fd, host_item *host,
 #ifdef EXPERIMENTAL_OCSP
     uschar *require_ocsp ARG_UNUSED,
 #endif
-    int dh_min_bits, int timeout)
+    int dh_min_bits, int timeout,
+    uschar *verify_hosts, uschar *try_verify_hosts)
 {
 int rc;
 const char *error;
diff --git a/src/src/tls-openssl.c b/src/src/tls-openssl.c
index 2b8d798c9..a9adb6134 100644
--- a/src/src/tls-openssl.c
+++ b/src/src/tls-openssl.c
@@ -1504,6 +1504,8 @@ Argument:
   dh_min_bits      minimum number of bits acceptable in server's DH prime
                    (unused in OpenSSL)
   timeout          startup timeout
+  verify_hosts     mandatory client verification 
+  try_verify_hosts optional client verification
 
 Returns:           OK on success
                    FAIL otherwise - note that tls_error() will not give DEFER
@@ -1518,7 +1520,8 @@ tls_client_start(int fd, host_item *host, address_item *addr,
 #ifdef EXPERIMENTAL_OCSP
   uschar *hosts_require_ocsp,
 #endif
-  int dh_min_bits ARG_UNUSED, int timeout)
+  int dh_min_bits ARG_UNUSED, int timeout,
+  uschar *verify_hosts, uschar *try_verify_hosts)
 {
 static uschar txt[256];
 uschar *expciphers;
@@ -1556,8 +1559,22 @@ if (expciphers != NULL)
     return tls_error(US"SSL_CTX_set_cipher_list", host, NULL);
   }
 
-rc = setup_certs(client_ctx, verify_certs, crl, host, FALSE, verify_callback_client);
-if (rc != OK) return rc;
+/* stick to the old behaviour for compatibility if tls_verify_certificates is 
+   set but both tls_verify_hosts and tls_try_verify_hosts is not set. Check only 
+   the specified host patterns if one of them is defined */
+if (((verify_hosts == NULL) && (try_verify_hosts == NULL)) ||
+    (verify_check_host(&verify_hosts) == OK))
+  {
+  rc = setup_certs(client_ctx, verify_certs, crl, host, FALSE, verify_callback_client);
+  if (rc != OK) return rc;
+  client_verify_optional = FALSE;
+  }
+else if (verify_check_host(&try_verify_hosts) == OK)
+  {
+  rc = setup_certs(client_ctx, verify_certs, crl, host, TRUE, verify_callback_client);
+  if (rc != OK) return rc;
+  client_verify_optional = TRUE;
+  }
 
 if ((client_ssl = SSL_new(client_ctx)) == NULL) return tls_error(US"SSL_new", host, NULL);
 SSL_set_session_id_context(client_ssl, sid_ctx, Ustrlen(sid_ctx));
diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index a77e472d6..938844799 100644
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -153,8 +153,12 @@ optionlist smtp_transport_options[] = {
       (void *)offsetof(smtp_transport_options_block, tls_sni) },
   { "tls_tempfail_tryclear", opt_bool,
       (void *)offsetof(smtp_transport_options_block, tls_tempfail_tryclear) },
+  { "tls_try_verify_hosts", opt_stringptr,
+      (void *)offsetof(smtp_transport_options_block, tls_try_verify_hosts) },
   { "tls_verify_certificates", opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, tls_verify_certificates) }
+      (void *)offsetof(smtp_transport_options_block, tls_verify_certificates) },
+  { "tls_verify_hosts",     opt_stringptr,
+      (void *)offsetof(smtp_transport_options_block, tls_verify_hosts) }
 #endif
 #ifdef EXPERIMENTAL_TPDA
  ,{ "tpda_host_defer_action", opt_stringptr,
@@ -227,7 +231,9 @@ smtp_transport_options_block smtp_transport_option_defaults = {
   NULL,                /* tls_verify_certificates */
   EXIM_CLIENT_DH_DEFAULT_MIN_BITS,
                        /* tls_dh_min_bits */
-  TRUE                 /* tls_tempfail_tryclear */
+  TRUE,                /* tls_tempfail_tryclear */
+  NULL,                /* tls_verify_hosts */
+  NULL                 /* tls_try_verify_hosts */
 #endif
 #ifndef DISABLE_DKIM
  ,NULL,                /* dkim_canon */
@@ -1446,7 +1452,9 @@ if (tls_offered && !suppress_tls &&
       ob->hosts_require_ocsp,
 #endif
       ob->tls_dh_min_bits,
-      ob->command_timeout);
+      ob->command_timeout,
+      ob->tls_verify_hosts,
+      ob->tls_try_verify_hosts);
 
     /* TLS negotiation failed; give an error. From outside, this function may
     be called again to try in clear on a new connection, if the options permit
diff --git a/src/src/transports/smtp.h b/src/src/transports/smtp.h
index 6c22e4b96..fc4e014c1 100644
--- a/src/src/transports/smtp.h
+++ b/src/src/transports/smtp.h
@@ -64,6 +64,8 @@ typedef struct {
   uschar *tls_verify_certificates;
   int     tls_dh_min_bits;
   BOOL    tls_tempfail_tryclear;
+  uschar *tls_verify_hosts;
+  uschar *tls_try_verify_hosts;
   #endif
   #ifndef DISABLE_DKIM
   uschar *dkim_domain;
diff --git a/src/src/verify.c b/src/src/verify.c
index 3c464fe93..cd91b0560 100644
--- a/src/src/verify.c
+++ b/src/src/verify.c
@@ -641,7 +641,8 @@ else
 #ifdef EXPERIMENTAL_OCSP
   	 ob->hosts_require_ocsp,
 #endif
-	 ob->tls_dh_min_bits,         callout);
+	 ob->tls_dh_min_bits, callout,
+         ob->tls_verify_hosts, ob->tls_try_verify_hosts);
 
         /* TLS negotiation failed; give an error.  Try in clear on a new connection,
            if the options permit it for this host. */
diff --git a/test/confs/2112 b/test/confs/2112
index 78733513e..242d2d67c 100644
--- a/test/confs/2112
+++ b/test/confs/2112
@@ -1,4 +1,5 @@
 # Exim test configuration 2112
+# TLS client: verify certificate from server - fails
 
 SERVER=
 
@@ -35,18 +36,53 @@ tls_verify_certificates = ${if eq {SERVER}{server}{DIR/aux-fixed/cert2}fail}
 
 begin routers
 
-client:
+server_dump:
+  driver = redirect
+  condition = ${if eq {SERVER}{server}{yes}{no}}
+  data = :blackhole:
+
+client_x:
+  driver = accept
+  local_parts = userx
+  retry_use_local_part
+  transport = send_to_server_failcert
+  errors_to = ""
+
+client_y:
+  driver = accept
+  local_parts = usery
+  retry_use_local_part
+  transport = send_to_server_retry
+
+client_z:
   driver = accept
-  condition = ${if eq {SERVER}{server}{no}{yes}}
+  local_parts = userz
   retry_use_local_part
-  transport = send_to_server
+  transport = send_to_server_crypt
+
+client_q:
+  driver = accept
+  local_parts = userq
+  retry_use_local_part
+  transport = send_to_server_req_fail
 
 
 # ----- Transports -----
 
 begin transports
 
-send_to_server:
+# this will fail to verify the cert at HOSTIPV4 so fail the crypt requirement
+send_to_server_failcert:
+  driver = smtp
+  allow_localhost
+  hosts = HOSTIPV4
+  hosts_require_tls = HOSTIPV4
+  port = PORT_D
+  tls_certificate = DIR/aux-fixed/cert2
+  tls_verify_certificates = DIR/aux-fixed/cert2
+
+# this will fail to verify the cert at HOSTIPV4 so fail the crypt, then retry on 127.1; ok
+send_to_server_retry:
   driver = smtp
   allow_localhost
   hosts = HOSTIPV4 : 127.0.0.1
@@ -56,4 +92,25 @@ send_to_server:
   tls_verify_certificates = \
     ${if eq{$host_address}{127.0.0.1}{DIR/aux-fixed/cert1}{DIR/aux-fixed/cert2}}
 
+# this will fail to verify the cert but continue unverified though cypted
+send_to_server_crypt:
+  driver = smtp
+  allow_localhost
+  hosts = HOSTIPV4
+  hosts_require_tls = HOSTIPV4
+  port = PORT_D
+  tls_certificate = DIR/aux-fixed/cert2
+  tls_verify_certificates = DIR/aux-fixed/cert2
+  tls_try_verify_hosts = *
+
+# this will fail to verify the cert at HOSTIPV4 and fallback to unencrypted
+send_to_server_req_fail:
+  driver = smtp
+  allow_localhost
+  hosts = HOSTIPV4
+  port = PORT_D
+  tls_certificate = DIR/aux-fixed/cert2
+  tls_verify_certificates = DIR/aux-fixed/cert2
+  tls_verify_hosts = *
+
 # End
diff --git a/test/log/2112 b/test/log/2112
index bee2f6fe3..3f77e65ea 100644
--- a/test/log/2112
+++ b/test/log/2112
@@ -1,13 +1,36 @@
 1999-03-02 09:44:33 10HmaX-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss
+1999-03-02 09:44:33 10HmaY-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss
+1999-03-02 09:44:33 10HmaZ-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss
+1999-03-02 09:44:33 10HmbA-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss
 1999-03-02 09:44:33 Start queue run: pid=pppp -qf
 1999-03-02 09:44:33 10HmaX-0005vi-00 SSL verify error: depth=0 error=self signed certificate cert=/C=UK/O=The Exim Maintainers/OU=Test Suite/CN=Phil Pennock
 1999-03-02 09:44:33 10HmaX-0005vi-00 TLS error on connection to ip4.ip4.ip4.ip4 [ip4.ip4.ip4.ip4] (SSL_connect): error: <<detail omitted>>
-1999-03-02 09:44:33 10HmaX-0005vi-00 => userx@test.ex R=client T=send_to_server H=127.0.0.1 [127.0.0.1] X=TLSv1:AES256-SHA:256 CV=yes DN="/C=UK/O=The Exim Maintainers/OU=Test Suite/CN=Phil Pennock" C="250 OK id=10HmaY-0005vi-00"
+1999-03-02 09:44:33 10HmaX-0005vi-00 == userx@test.ex R=client_x T=send_to_server_failcert defer (-37): failure while setting up TLS session
+1999-03-02 09:44:33 10HmaX-0005vi-00 ** userx@test.ex: retry timeout exceeded
+1999-03-02 09:44:33 10HmaX-0005vi-00 userx@test.ex: error ignored
 1999-03-02 09:44:33 10HmaX-0005vi-00 Completed
+1999-03-02 09:44:33 10HmaY-0005vi-00 SSL verify error: depth=0 error=self signed certificate cert=/C=UK/O=The Exim Maintainers/OU=Test Suite/CN=Phil Pennock
+1999-03-02 09:44:33 10HmaY-0005vi-00 TLS error on connection to ip4.ip4.ip4.ip4 [ip4.ip4.ip4.ip4] (SSL_connect): error: <<detail omitted>>
+1999-03-02 09:44:33 10HmaY-0005vi-00 => usery@test.ex R=client_y T=send_to_server_retry H=127.0.0.1 [127.0.0.1] X=TLSv1:AES256-SHA:256 CV=yes DN="/C=UK/O=The Exim Maintainers/OU=Test Suite/CN=Phil Pennock" C="250 OK id=10HmbB-0005vi-00"
+1999-03-02 09:44:33 10HmaY-0005vi-00 Completed
+1999-03-02 09:44:33 10HmaZ-0005vi-00 SSL verify error: depth=0 error=self signed certificate cert=/C=UK/O=The Exim Maintainers/OU=Test Suite/CN=Phil Pennock
+1999-03-02 09:44:33 10HmaZ-0005vi-00 => userz@test.ex R=client_z T=send_to_server_crypt H=ip4.ip4.ip4.ip4 [ip4.ip4.ip4.ip4] X=TLSv1:AES256-SHA:256 CV=no DN="/C=UK/O=The Exim Maintainers/OU=Test Suite/CN=Phil Pennock" C="250 OK id=10HmbC-0005vi-00"
+1999-03-02 09:44:33 10HmaZ-0005vi-00 Completed
+1999-03-02 09:44:33 10HmbA-0005vi-00 SSL verify error: depth=0 error=self signed certificate cert=/C=UK/O=The Exim Maintainers/OU=Test Suite/CN=Phil Pennock
+1999-03-02 09:44:33 10HmbA-0005vi-00 TLS error on connection to ip4.ip4.ip4.ip4 [ip4.ip4.ip4.ip4] (SSL_connect): error: <<detail omitted>>
+1999-03-02 09:44:33 10HmbA-0005vi-00 TLS session failure: delivering unencrypted to ip4.ip4.ip4.ip4 [ip4.ip4.ip4.ip4] (not in hosts_require_tls)
+1999-03-02 09:44:33 10HmbA-0005vi-00 => userq@test.ex R=client_q T=send_to_server_req_fail H=ip4.ip4.ip4.ip4 [ip4.ip4.ip4.ip4] C="250 OK id=10HmbD-0005vi-00"
+1999-03-02 09:44:33 10HmbA-0005vi-00 Completed
 1999-03-02 09:44:33 End queue run: pid=pppp -qf
 
 ******** SERVER ********
 1999-03-02 09:44:33 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTP on port 1225
 1999-03-02 09:44:33 TLS error on connection from the.local.host.name (myhost.test.ex) [ip4.ip4.ip4.ip4] (SSL_accept): error: <<detail omitted>>
 1999-03-02 09:44:33 TLS client disconnected cleanly (rejected our certificate?)
-1999-03-02 09:44:33 10HmaY-0005vi-00 <= CALLER@myhost.test.ex H=localhost (myhost.test.ex) [127.0.0.1] P=esmtps X=TLSv1:AES256-SHA:256 CV=yes DN="/C=UK/O=The Exim Maintainers/OU=Test Suite/CN=Phil Pennock" S=sss id=E10HmaX-0005vi-00@myhost.test.ex
+1999-03-02 09:44:33 TLS error on connection from the.local.host.name (myhost.test.ex) [ip4.ip4.ip4.ip4] (SSL_accept): error: <<detail omitted>>
+1999-03-02 09:44:33 TLS client disconnected cleanly (rejected our certificate?)
+1999-03-02 09:44:33 10HmbB-0005vi-00 <= CALLER@myhost.test.ex H=localhost (myhost.test.ex) [127.0.0.1] P=esmtps X=TLSv1:AES256-SHA:256 CV=yes DN="/C=UK/O=The Exim Maintainers/OU=Test Suite/CN=Phil Pennock" S=sss id=E10HmaY-0005vi-00@myhost.test.ex
+1999-03-02 09:44:33 10HmbC-0005vi-00 <= CALLER@myhost.test.ex H=the.local.host.name (myhost.test.ex) [ip4.ip4.ip4.ip4] P=esmtps X=TLSv1:AES256-SHA:256 CV=yes DN="/C=UK/O=The Exim Maintainers/OU=Test Suite/CN=Phil Pennock" S=sss id=E10HmaZ-0005vi-00@myhost.test.ex
+1999-03-02 09:44:33 TLS error on connection from the.local.host.name (myhost.test.ex) [ip4.ip4.ip4.ip4] (SSL_accept): error: <<detail omitted>>
+1999-03-02 09:44:33 TLS client disconnected cleanly (rejected our certificate?)
+1999-03-02 09:44:33 10HmbD-0005vi-00 <= CALLER@myhost.test.ex H=the.local.host.name (myhost.test.ex) [ip4.ip4.ip4.ip4] P=esmtp S=sss id=E10HmbA-0005vi-00@myhost.test.ex
diff --git a/test/scripts/2100-OpenSSL/2112 b/test/scripts/2100-OpenSSL/2112
index 4793929bc..98ea4cb17 100644
--- a/test/scripts/2100-OpenSSL/2112
+++ b/test/scripts/2100-OpenSSL/2112
@@ -4,6 +4,15 @@ exim -DSERVER=server -bd -oX PORT_D
 exim userx@test.ex
 Testing
 ****
+exim usery@test.ex
+Testing
+****
+exim userz@test.ex
+Testing
+****
+exim userq@test.ex
+Testing
+****
 exim -qf
 ****
 killdaemon