Pipeline QUIT after data

13 files changed, 329 insertions, 138 deletions

diff --git a/src/src/daemon.c b/src/src/daemon.c
index 50c202c56..52e6dcab1 100644
--- a/src/src/daemon.c
+++ b/src/src/daemon.c
@@ -554,7 +554,7 @@ if (pid == 0)
         }
       if (message_id[0] == 0) continue;   /* No message was accepted */
       }
-    else
+    else				/* bad smtp_setup_msg() */
       {
       if (smtp_out)
 	{
@@ -674,15 +674,15 @@ if (pid == 0)
       {
       pid_t dpid;
 
-      /* Before forking, ensure that the C output buffer is flushed. Otherwise
-      anything that it in it will get duplicated, leading to duplicate copies
-      of the pending output. */
-
-      mac_smtp_fflush();
+      /* We used to flush smtp_out before forking so that buffered data was not
+      duplicated, but now we want to pipeline the responses for data and quit.
+      Instead, hard-close the fd underlying smtp_out right after fork to discard
+      the data buffer. */
 
       if ((dpid = exim_fork(US"daemon-accept-delivery")) == 0)
         {
         (void)fclose(smtp_in);
+	(void)close(fileno(smtp_out));
         (void)fclose(smtp_out);
 
         /* Don't ever molest the parent's SSL connection, but do clean up
diff --git a/src/src/deliver.c b/src/src/deliver.c
index 833970c5a..29e2b719d 100644
--- a/src/src/deliver.c
+++ b/src/src/deliver.c
@@ -8549,52 +8549,6 @@ f.tcp_fastopen_ok = TRUE;
 }
 
 
-uschar *
-deliver_get_sender_address (uschar * id)
-{
-int rc;
-uschar * new_sender_address,
-       * save_sender_address;
-BOOL save_qr = f.queue_running;
-uschar * spoolname;
-
-/* make spool_open_datafile non-noisy on fail */
-
-f.queue_running = TRUE;
-
-/* Side effect: message_subdir is set for the (possibly split) spool directory */
-
-deliver_datafile = spool_open_datafile(id);
-f.queue_running = save_qr;
-if (deliver_datafile < 0)
-  return NULL;
-
-/* Save and restore the global sender_address.  I'm not sure if we should
-not save/restore all the other global variables too, because
-spool_read_header() may change all of them. But OTOH, when this
-deliver_get_sender_address() gets called, the current message is done
-already and nobody needs the globals anymore. (HS12, 2015-08-21) */
-
-spoolname = string_sprintf("%s-H", id);
-save_sender_address = sender_address;
-
-rc = spool_read_header(spoolname, TRUE, TRUE);
-
-new_sender_address = sender_address;
-sender_address = save_sender_address;
-
-if (rc != spool_read_OK)
-  return NULL;
-
-assert(new_sender_address);
-
-(void)close(deliver_datafile);
-deliver_datafile = -1;
-
-return new_sender_address;
-}
-
-
 
 void
 delivery_re_exec(int exec_type)
diff --git a/src/src/functions.h b/src/src/functions.h
index 38309e7dd..5ffb23d1e 100644
--- a/src/src/functions.h
+++ b/src/src/functions.h
@@ -75,6 +75,7 @@ extern BOOL    tls_openssl_options_parse(uschar *, long *);
 # endif
 extern int     tls_read(void *, uschar *, size_t);
 extern int     tls_server_start(uschar **);
+extern void    tls_shutdown_wr(void *);
 extern BOOL    tls_smtp_buffered(void);
 extern int     tls_ungetc(int);
 #if defined(EXIM_HAVE_INOTIFY) || defined(EXIM_HAVE_KEVENT)
@@ -204,7 +205,6 @@ extern void    deliver_set_expansions(address_item *);
 extern int     deliver_split_address(address_item *);
 extern void    deliver_succeeded(address_item *);
 
-extern uschar *deliver_get_sender_address (uschar *id);
 extern void    delivery_re_exec(int);
 
 extern void    die_tainted(const uschar *, const uschar *, int);
@@ -514,6 +514,7 @@ extern BOOL    spool_move_message(uschar *, uschar *, uschar *, uschar *);
 extern int     spool_open_datafile(uschar *);
 extern int     spool_open_temp(uschar *);
 extern int     spool_read_header(uschar *, BOOL, BOOL);
+extern uschar *spool_sender_from_msgid(const uschar *);
 extern int     spool_write_header(uschar *, int, uschar **);
 extern int     stdin_getc(unsigned);
 extern int     stdin_feof(void);
diff --git a/src/src/macros.h b/src/src/macros.h
index 68470a9f1..0b647569b 100644
--- a/src/src/macros.h
+++ b/src/src/macros.h
@@ -866,18 +866,19 @@ enum {
 
 /* Options for transport_write_message */
 
-#define topt_add_return_path    0x001
-#define topt_add_delivery_date  0x002
-#define topt_add_envelope_to    0x004
-#define topt_use_crlf           0x008  /* Terminate lines with CRLF */
-#define topt_end_dot            0x010  /* Send terminating dot line */
-#define topt_no_headers         0x020  /* Omit headers */
-#define topt_no_body            0x040  /* Omit body */
-#define topt_escape_headers     0x080  /* Apply escape check to headers */
-#define topt_use_bdat		0x100  /* prepend chunks with RFC3030 BDAT header */
-#define topt_output_string	0x200  /* create string rather than write to fd */
-#define topt_continuation	0x400  /* do not reset buffer */
-#define topt_not_socket		0x800  /* cannot do socket-only syscalls */
+#define topt_add_return_path    0x0001
+#define topt_add_delivery_date  0x0002
+#define topt_add_envelope_to    0x0004
+#define topt_escape_headers     0x0008	/* Apply escape check to headers */
+#define topt_use_crlf           0x0010	/* Terminate lines with CRLF */
+#define topt_no_headers         0x0020	/* Omit headers */
+#define topt_no_body            0x0040	/* Omit body */
+#define topt_end_dot            0x0080	/* Send terminating dot line */
+#define topt_no_flush		0x0100	/* more data expected after message (eg QUIT) */
+#define topt_use_bdat		0x0200	/* prepend chunks with RFC3030 BDAT header */
+#define topt_output_string	0x0400	/* create string rather than write to fd */
+#define topt_continuation	0x0800	/* do not reset buffer */
+#define topt_not_socket		0x1000	/* cannot do socket-only syscalls */
 
 /* Options for smtp_write_command */
 
diff --git a/src/src/receive.c b/src/src/receive.c
index 00c431fc8..3e950ffc6 100644
--- a/src/src/receive.c
+++ b/src/src/receive.c
@@ -4177,12 +4177,10 @@ response, but the chance of this happening should be small. */
 if (smtp_input && sender_host_address && !f.sender_host_notsocket &&
     !receive_smtp_buffered())
   {
-  struct timeval tv;
+  struct timeval tv = {.tv_sec = 0, .tv_usec = 0};
   fd_set select_check;
   FD_ZERO(&select_check);
   FD_SET(fileno(smtp_in), &select_check);
-  tv.tv_sec = 0;
-  tv.tv_usec = 0;
 
   if (select(fileno(smtp_in) + 1, &select_check, NULL, NULL, &tv) != 0)
     {
@@ -4375,12 +4373,17 @@ if (smtp_input)
 
       else if (chunking_state > CHUNKING_OFFERED)
 	{
-        smtp_printf("250- %u byte chunk, total %d\r\n250 OK id=%s\r\n", FALSE,
+	/* If there is more input waiting, no need to flush (probably the client
+	pipelined QUIT after data).  We check only the in-process buffer, not
+	the socket. */
+
+        smtp_printf("250- %u byte chunk, total %d\r\n250 OK id=%s\r\n",
+	    receive_smtp_buffered(),
 	    chunking_datasize, message_size+message_linecount, message_id);
 	chunking_state = CHUNKING_OFFERED;
 	}
       else
-        smtp_printf("250 OK id=%s\r\n", FALSE, message_id);
+        smtp_printf("250 OK id=%s\r\n", receive_smtp_buffered(), message_id);
 
       if (host_checking)
         fprintf(stdout,
diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c
index 14dd11498..03085c9e4 100644
--- a/src/src/smtp_in.c
+++ b/src/src/smtp_in.c
@@ -347,7 +347,7 @@ wouldblock_reading(void)
 {
 int fd, rc;
 fd_set fds;
-struct timeval tzero;
+struct timeval tzero = {.tv_sec = 0, .tv_usec = 0};
 
 #ifndef DISABLE_TLS
 if (tls_in.active.sock >= 0)
@@ -360,8 +360,6 @@ if (smtp_inptr < smtp_inend)
 fd = fileno(smtp_in);
 FD_ZERO(&fds);
 FD_SET(fd, &fds);
-tzero.tv_sec = 0;
-tzero.tv_usec = 0;
 rc = select(fd + 1, (SELECT_ARG2_TYPE *)&fds, NULL, NULL, &tzero);
 
 if (rc <= 0) return TRUE;     /* Not ready to read */
@@ -587,6 +585,8 @@ smtp_get_cache(void)
 {
 #ifndef DISABLE_DKIM
 int n = smtp_inend - smtp_inptr;
+if (chunking_state == CHUNKING_LAST && chunking_data_left < n)
+  n = chunking_data_left;
 if (n > 0)
   dkim_exim_verify_feed(smtp_inptr, n);
 #endif
@@ -3844,11 +3844,24 @@ else
   smtp_printf("221 %s closing connection\r\n", FALSE, smtp_active_hostname);
 
 #ifndef DISABLE_TLS
-tls_close(NULL, TLS_SHUTDOWN_NOWAIT);
+tls_close(NULL, TLS_SHUTDOWN_WAIT);
 #endif
 
 log_write(L_smtp_connection, LOG_MAIN, "%s closed by QUIT",
   smtp_get_connection_info());
+
+/* Pause, hoping client will FIN first so that they get the TIME_WAIT.
+The socket should become readble (though with no data) */
+
+  {
+  int fd = fileno(smtp_in);
+  fd_set fds;
+  struct timeval t_limit = {.tv_sec = 0, .tv_usec = 200*1000};
+
+  FD_ZERO(&fds);
+  FD_SET(fd, &fds);
+  (void) select(fd + 1, (SELECT_ARG2_TYPE *)&fds, NULL, NULL, &t_limit);
+  }
 }
 
 
diff --git a/src/src/smtp_out.c b/src/src/smtp_out.c
index 2b03c03f3..2a4497488 100644
--- a/src/src/smtp_out.c
+++ b/src/src/smtp_out.c
@@ -659,8 +659,9 @@ if (format)
     while (*p) *p++ = '*';
     }
 
-  HDEBUG(D_transport|D_acl|D_v)
-    debug_printf_indent("  SMTP%c> %s\n", ">+|"[mode], big_buffer);
+  HDEBUG(D_transport|D_acl|D_v) debug_printf_indent("  SMTP%c> %s\n",
+    mode == SCMD_BUFFER ? '|' : mode == SCMD_MORE ? '+' : '>',
+    big_buffer);
   }
 
 if (mode != SCMD_BUFFER)
diff --git a/src/src/spool_in.c b/src/src/spool_in.c
index 022ac02bc..9794e93d1 100644
--- a/src/src/spool_in.c
+++ b/src/src/spool_in.c
@@ -993,6 +993,47 @@ errno = ERRNO_SPOOLFORMAT;
 return inheader? spool_read_hdrerror : spool_read_enverror;
 }
 
+
+#ifndef COMPILE_UTILITY
+/* Read out just the (envelope) sender string from the spool -H file.
+Remove the <> wrap and return it in allocated store.  Return NULL on error.
+
+We assume that message_subdir is already set.
+*/
+
+uschar *
+spool_sender_from_msgid(const uschar * id)
+{
+uschar * name = string_sprintf("%s-H", id);
+FILE * fp;
+int n;
+uschar * yield = NULL;
+
+if (!(fp = Ufopen(spool_fname(US"input", message_subdir, name, US""), "rb")))
+  return NULL;
+
+DEBUG(D_deliver) debug_printf_indent("reading spool file %s\n", name);
+
+/* Skip the line with the copy of the filename, then the line with login/uid/gid.
+Read the next line, which should be the envelope sender.
+Do basic validation on that. */
+
+if (  Ufgets(big_buffer, big_buffer_size, fp) != NULL
+   && Ufgets(big_buffer, big_buffer_size, fp) != NULL
+   && Ufgets(big_buffer, big_buffer_size, fp) != NULL
+   && (n = Ustrlen(big_buffer)) >= 3
+   && big_buffer[0] == '<' && big_buffer[n-2] == '>'
+   )
+  {
+  yield = store_get(n-2, TRUE);	/* tainted */
+  Ustrncpy(yield, big_buffer+1, n-3);
+  yield[n-3] = 0;
+  }
+fclose(fp);
+return yield;
+}
+#endif  /* COMPILE_UTILITY */
+
 /* vi: aw ai sw=2
 */
 /* End of spool_in.c */
diff --git a/src/src/tls-gnu.c b/src/src/tls-gnu.c
index 46547d732..89e3ef2f6 100644
--- a/src/src/tls-gnu.c
+++ b/src/src/tls-gnu.c
@@ -3489,6 +3489,25 @@ return TRUE;
 
 
 
+/*
+Arguments:
+  ct_ctx	client TLS context pointer, or NULL for the one global server context
+*/
+
+void
+tls_shutdown_wr(void * ct_ctx)
+{
+exim_gnutls_state_st * state = ct_ctx ? ct_ctx : &state_server;
+tls_support * tlsp = state->tlsp;
+
+if (!tlsp || tlsp->active.sock < 0) return;  /* TLS was not active */
+
+tls_write(ct_ctx, NULL, 0, FALSE);	/* flush write buffer */
+
+HDEBUG(D_transport|D_tls|D_acl|D_v) debug_printf_indent("  SMTP(TLS shutdown)>>\n");
+gnutls_bye(state->session, GNUTLS_SHUT_WR);
+}
+
 /*************************************************
 *         Close down a TLS session               *
 *************************************************/
@@ -3499,29 +3518,30 @@ would tamper with the TLS session in the parent process).
 
 Arguments:
   ct_ctx	client context pointer, or NULL for the one global server context
-  shutdown	1 if TLS close-alert is to be sent,
-		2 if also response to be waited for
+  do_shutdown	0 no data-flush or TLS close-alert
+		1 if TLS close-alert is to be sent,
+		2 if also response to be waited for (2s timeout)
 
 Returns:     nothing
 */
 
 void
-tls_close(void * ct_ctx, int shutdown)
+tls_close(void * ct_ctx, int do_shutdown)
 {
 exim_gnutls_state_st * state = ct_ctx ? ct_ctx : &state_server;
 tls_support * tlsp = state->tlsp;
 
 if (!tlsp || tlsp->active.sock < 0) return;  /* TLS was not active */
 
-tls_write(ct_ctx, NULL, 0, FALSE);	/* flush write buffer */
-
-if (shutdown)
+if (do_shutdown)
   {
   DEBUG(D_tls) debug_printf("tls_close(): shutting down TLS%s\n",
-    shutdown > 1 ? " (with response-wait)" : "");
+    do_shutdown > 1 ? " (with response-wait)" : "");
+
+  tls_write(ct_ctx, NULL, 0, FALSE);	/* flush write buffer */
 
   ALARM(2);
-  gnutls_bye(state->session, shutdown > 1 ? GNUTLS_SHUT_RDWR : GNUTLS_SHUT_WR);
+  gnutls_bye(state->session, do_shutdown > 1 ? GNUTLS_SHUT_RDWR : GNUTLS_SHUT_WR);
   ALARM_CLR(0);
   }
 
@@ -3669,7 +3689,7 @@ return buf;
 
 
 void
-tls_get_cache()
+tls_get_cache(void)
 {
 #ifndef DISABLE_DKIM
 exim_gnutls_state_st * state = &state_server;
@@ -3688,8 +3708,6 @@ return state_server.xfer_buffer_lwm < state_server.xfer_buffer_hwm
 }
 
 
-
-
 /*************************************************
 *          Read bytes from TLS channel           *
 *************************************************/
diff --git a/src/src/tls-openssl.c b/src/src/tls-openssl.c
index 0d653a445..13b0c232f 100644
--- a/src/src/tls-openssl.c
+++ b/src/src/tls-openssl.c
@@ -3941,7 +3941,7 @@ return buf;
 
 
 void
-tls_get_cache()
+tls_get_cache(void)
 {
 #ifndef DISABLE_DKIM
 int n = ssl_xfer_buffer_hwm - ssl_xfer_buffer_lwm;
@@ -3955,7 +3955,7 @@ BOOL
 tls_could_read(void)
 {
 return ssl_xfer_buffer_lwm < ssl_xfer_buffer_hwm
-      || SSL_pending(state_server.lib_state.lib_ssl) > 0;
+    || SSL_pending(state_server.lib_state.lib_ssl) > 0;
 }
 
 
@@ -4120,6 +4120,32 @@ return olen;
 
 
 
+/*
+Arguments:
+  ct_ctx	client TLS context pointer, or NULL for the one global server context
+*/
+
+void
+tls_shutdown_wr(void * ct_ctx)
+{
+exim_openssl_client_tls_ctx * o_ctx = ct_ctx;
+SSL ** sslp = o_ctx ? &o_ctx->ssl : (SSL **) &state_server.lib_state.lib_ssl;
+int * fdp = o_ctx ? &tls_out.active.sock : &tls_in.active.sock;
+int rc;
+
+if (*fdp < 0) return;  /* TLS was not active */
+
+tls_write(ct_ctx, NULL, 0, FALSE);	/* flush write buffer */
+
+HDEBUG(D_transport|D_tls|D_acl|D_v) debug_printf_indent("  SMTP(TLS shutdown)>>\n");
+rc = SSL_shutdown(*sslp);
+if (rc < 0) DEBUG(D_tls)
+  {
+  ERR_error_string_n(ERR_get_error(), ssl_errstring, sizeof(ssl_errstring));
+  debug_printf("SSL_shutdown: %s\n", ssl_errstring);
+  }
+}
+
 /*************************************************
 *         Close down a TLS session               *
 *************************************************/
@@ -4130,7 +4156,8 @@ would tamper with the SSL session in the parent process).
 
 Arguments:
   ct_ctx	client TLS context pointer, or NULL for the one global server context
-  shutdown	1 if TLS close-alert is to be sent,
+  do_shutdown	0 no data-flush or TLS close-alert
+		1 if TLS close-alert is to be sent,
  		2 if also response to be waited for
 
 Returns:     nothing
@@ -4139,24 +4166,24 @@ Used by both server-side and client-side TLS.
 */
 
 void
-tls_close(void * ct_ctx, int shutdown)
+tls_close(void * ct_ctx, int do_shutdown)
 {
 exim_openssl_client_tls_ctx * o_ctx = ct_ctx;
-SSL **sslp = o_ctx ? &o_ctx->ssl : (SSL **) &state_server.lib_state.lib_ssl;
-int *fdp = o_ctx ? &tls_out.active.sock : &tls_in.active.sock;
+SSL ** sslp = o_ctx ? &o_ctx->ssl : (SSL **) &state_server.lib_state.lib_ssl;
+int * fdp = o_ctx ? &tls_out.active.sock : &tls_in.active.sock;
 
 if (*fdp < 0) return;  /* TLS was not active */
 
-tls_write(ct_ctx, NULL, 0, FALSE);	/* flush write buffer */
-
-if (shutdown)
+if (do_shutdown)
   {
   int rc;
   DEBUG(D_tls) debug_printf("tls_close(): shutting down TLS%s\n",
-    shutdown > 1 ? " (with response-wait)" : "");
+    do_shutdown > 1 ? " (with response-wait)" : "");
+
+  tls_write(ct_ctx, NULL, 0, FALSE);	/* flush write buffer */
 
   if (  (rc = SSL_shutdown(*sslp)) == 0	/* send "close notify" alert */
-     && shutdown > 1)
+     && do_shutdown > 1)
     {
     ALARM(2);
     rc = SSL_shutdown(*sslp);		/* wait for response */
diff --git a/src/src/transport.c b/src/src/transport.c
index c2062e633..31edb9692 100644
--- a/src/src/transport.c
+++ b/src/src/transport.c
@@ -886,7 +886,7 @@ transport_write_timeout non-zero.
 
 Arguments:
   tctx
-    (fd, msg)		Either and fd, to write the message to,
+    (fd, msg)		Either an fd, to write the message to,
 			or a string: if null write message to allocated space
 			otherwire take content as headers.
     addr                (chain of) addresses (for extra headers), or NULL;
@@ -905,6 +905,7 @@ Arguments:
       add_delivery_date     if TRUE, add a "delivery-date" header
       use_crlf              if TRUE, turn NL into CR LF
       end_dot               if TRUE, send a terminating "." line at the end
+      no_flush		    if TRUE, do not flush at end
       no_headers            if TRUE, omit the headers
       no_body               if TRUE, omit the body
     check_string          a string to check for at the start of lines, or NULL
@@ -1161,8 +1162,9 @@ if (tctx->options & topt_end_dot && !write_chunk(tctx, US".\n", 2))
 
 /* Write out any remaining data in the buffer before returning. */
 
-return (len = chunk_ptr - deliver_out_buffer) <= 0 ||
-  transport_write_block(tctx, deliver_out_buffer, len, FALSE);
+return (len = chunk_ptr - deliver_out_buffer) <= 0
+  || transport_write_block(tctx, deliver_out_buffer, len,
+			    !!(tctx->options & topt_no_flush));
 }
 
 
@@ -1260,7 +1262,7 @@ if ((write_pid = exim_fork(US"tpt-filter-writer")) == 0)
 
   tctx->u.fd = fd_write;
   tctx->check_string = tctx->escape_string = NULL;
-  tctx->options &= ~(topt_use_crlf | topt_end_dot | topt_use_bdat);
+  tctx->options &= ~(topt_use_crlf | topt_end_dot | topt_use_bdat | topt_no_flush);
 
   rc = internal_transport_write_message(tctx, size_limit);
 
@@ -1599,7 +1601,8 @@ for (host_item * host = hostlist; host; host = host->next)
   /* Update the database */
 
   dbfn_write(dbm_file, host->name, host_record, sizeof(dbdata_wait) + host_length);
-  DEBUG(D_transport) debug_printf("added to list for %s\n", host->name);
+  DEBUG(D_transport) debug_printf("added %.*s to queue for %s\n",
+				  MESSAGE_ID_LENGTH, message_id, host->name);
   }
 
 /* All now done */
diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index 95fa6daa9..13f617cbb 100644
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -1637,8 +1637,9 @@ uschar * message_local_identity,
 current_local_identity =
   smtp_local_identity(s_compare->current_sender_address, s_compare->tblock);
 
-if (!(new_sender_address = deliver_get_sender_address(message_id)))
-    return FALSE;
+if (!(new_sender_address = spool_sender_from_msgid(message_id)))
+  return FALSE;
+
 
 message_local_identity =
   smtp_local_identity(new_sender_address, s_compare->tblock);
@@ -2516,6 +2517,7 @@ if (  smtp_peer_options & OPTION_TLS
       sx->send_quit = FALSE;
       goto TLS_FAILED;
       }
+    sx->send_tlsclose = TRUE;
 
     /* TLS session is set up.  Check the inblock fill level.  If there is
     content then as we have not yet done a tls read it must have arrived before
@@ -2949,7 +2951,11 @@ if (sx->send_quit)
 #ifndef DISABLE_TLS
 if (sx->cctx.tls_ctx)
   {
-  tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_NOWAIT);
+  if (sx->send_tlsclose)
+    {
+    tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_NOWAIT);
+    sx->send_tlsclose = FALSE;
+    }
   sx->cctx.tls_ctx = NULL;
   }
 #endif
@@ -3328,6 +3334,7 @@ smtp_proxy_tls(void * ct_ctx, uschar * buf, size_t bsize, int * pfd,
 fd_set rfds, efds;
 int max_fd = MAX(pfd[0], tls_out.active.sock) + 1;
 int rc, i;
+BOOL send_tls_shutdown = TRUE;
 
 close(pfd[1]);
 if ((rc = exim_fork(US"tls-proxy")))
@@ -3372,8 +3379,8 @@ for (int fd_bits = 3; fd_bits; )
 
   /* handle inbound data */
   if (FD_ISSET(tls_out.active.sock, &rfds))
-    if ((rc = tls_read(ct_ctx, buf, bsize)) <= 0)
-      {
+    if ((rc = tls_read(ct_ctx, buf, bsize)) <= 0)	/* Expect -1 for EOF; */
+    {				    /* that reaps the TLS Close Notify record */
       fd_bits &= ~1;
       FD_CLR(tls_out.active.sock, &rfds);
       shutdown(pfd[0], SHUT_WR);
@@ -3384,16 +3391,21 @@ for (int fd_bits = 3; fd_bits; )
       for (int nbytes = 0; rc - nbytes > 0; nbytes += i)
 	if ((i = write(pfd[0], buf + nbytes, rc - nbytes)) < 0) goto done;
       }
-  else if (fd_bits & 1)
-    FD_SET(tls_out.active.sock, &rfds);
 
-  /* handle outbound data */
+  /* Handle outbound data.  We cannot combine payload and the TLS-close
+  due to the limitations of the (pipe) channel feeding us. */
   if (FD_ISSET(pfd[0], &rfds))
     if ((rc = read(pfd[0], buf, bsize)) <= 0)
       {
-      fd_bits = 0;
-      tls_close(ct_ctx, TLS_SHUTDOWN_NOWAIT);
-      ct_ctx = NULL;
+      fd_bits &= ~2;
+      FD_CLR(pfd[0], &rfds);
+
+# ifdef EXIM_TCP_CORK  /* Use _CORK to get TLS Close Notify in FIN segment */
+      (void) setsockopt(tls_out.active.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on));
+# endif
+      tls_shutdown_wr(ct_ctx);
+      send_tls_shutdown = FALSE;
+      shutdown(tls_out.active.sock, SHUT_WR);
       }
     else
       {
@@ -3401,11 +3413,14 @@ for (int fd_bits = 3; fd_bits; )
 	if ((i = tls_write(ct_ctx, buf + nbytes, rc - nbytes, FALSE)) < 0)
 	  goto done;
       }
-  else if (fd_bits & 2)
-    FD_SET(pfd[0], &rfds);
+
+  if (fd_bits & 1) FD_SET(tls_out.active.sock, &rfds);
+  if (fd_bits & 2) FD_SET(pfd[0], &rfds);
   }
 
 done:
+  if (send_tls_shutdown) tls_close(ct_ctx, TLS_SHUTDOWN_NOWAIT);
+  ct_ctx = NULL;
   testharness_pause_ms(100);	/* let logging complete */
   exim_exit(EXIT_SUCCESS);
 }
@@ -3478,6 +3493,7 @@ smtp_context * sx = store_get(sizeof(*sx), TRUE);	/* tainted, for the data buffe
 #ifdef SUPPORT_DANE
 BOOL dane_held;
 #endif
+BOOL tcw_done = FALSE, tcw = FALSE;
 
 *message_defer = FALSE;
 
@@ -3767,6 +3783,44 @@ else
   report_time_since(&t0, US"dkim_exim_sign_init (delta)");
 # endif
   }
+#endif
+
+  /* See if we can pipeline QUIT.  Reasons not to are
+  - pipelining not active
+  - not ok to send quit
+  - errors in amtp transation responses
+  - more addrs to send for this message or this host
+  - more messages for this host
+  If we can, we want the message-write to not flush (the tail end of) its data out.  */
+
+  if (  sx->pipelining_used
+     && (sx->ok && sx->completed_addr || sx->peer_offered & OPTION_CHUNKING)
+     && sx->send_quit
+     && !(sx->first_addr || f.continue_more)
+     )
+    {
+    smtp_compare_t t_compare =
+      {.tblock = tblock, .current_sender_address = sender_address};
+
+    tcw_done = TRUE;
+    tcw =
+#ifndef DISABLE_TLS
+	   (  tls_out.active.sock < 0  &&  !continue_proxy_cipher
+           || verify_check_given_host(CUSS &ob->hosts_nopass_tls, host) != OK
+	   )
+        &&
+#endif
+           transport_check_waiting(tblock->name, host->name,
+             tblock->connection_max_messages, new_message_id,
+	     (oicf)smtp_are_same_identities, (void*)&t_compare);
+    if (!tcw)
+      {
+      HDEBUG(D_transport) debug_printf("will pipeline QUIT\n");
+      tctx.options |= topt_no_flush;
+      }
+    }
+
+#ifndef DISABLE_DKIM
   sx->ok = dkim_transport_write_message(&tctx, &ob->dkim, CUSS &message);
 #else
   sx->ok = transport_write_message(&tctx, 0);
@@ -3795,6 +3849,47 @@ else
 
   smtp_command = US"end of data";
 
+  /* If we can pipeline a QUIT with the data them send it now.  If a new message
+  for this host appeared in the queue while data was being sent, we will not see
+  it and it will have to wait for a queue run.  If there was one but another
+  thread took it, we might attempt to send it - but locking of spoolfiles will
+  detect that. Use _MORE to get QUIT in FIN segment. */
+
+  if (tcw_done && !tcw)
+    {
+    /*XXX jgh 2021/03/10 google et. al screwup.  G, at least, sends TCP FIN in response to TLS
+    close-notify.  Under TLS 1.3, violating RFC.
+    However, TLS 1.2 does not have half-close semantics. */
+
+    if (  sx->cctx.tls_ctx
+#if 0 && !defined(DISABLE_TLS)
+       && Ustrcmp(tls_out.ver, "TLS1.3") != 0
+#endif
+       )
+      {				/* Send QUIT now and not later */
+      (void)smtp_write_command(sx, SCMD_FLUSH, "QUIT\r\n");
+      sx->send_quit = FALSE;
+      }
+    else
+      {				/* add QUIT to the output buffer */
+      (void)smtp_write_command(sx, SCMD_MORE, "QUIT\r\n");
+      sx->send_quit = FALSE;	/* avoid sending it later */
+
+#ifndef DISABLE_TLS
+      if (sx->cctx.tls_ctx)	/* need to send TLS Cloe Notify */
+	{
+# ifdef EXIM_TCP_CORK		/* Use _CORK to get Close Notify in FIN segment */
+	(void) setsockopt(sx->cctx.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on));
+# endif
+	tls_shutdown_wr(sx->cctx.tls_ctx);
+	sx->send_tlsclose = FALSE;	/* avoid later repeat */
+	}
+#endif
+      HDEBUG(D_transport|D_acl|D_v) debug_printf_indent("  SMTP(shutdown)>>\n");
+      shutdown(sx->cctx.sock, SHUT_WR);	/* flush output buffer, with TCP FIN */
+      }
+    }
+
   if (sx->peer_offered & OPTION_CHUNKING && sx->cmd_count > 1)
     {
     /* Reap any outstanding MAIL & RCPT commands, but not a DATA-go-ahead */
@@ -4069,7 +4164,8 @@ if (!sx->ok)
     {
     save_errno = errno;
     message = NULL;
-    sx->send_quit = check_response(host, &save_errno, addrlist->more_errno,
+    /* Clear send_quit flag if needed.  Do not set. */
+    sx->send_quit &= check_response(host, &save_errno, addrlist->more_errno,
       sx->buffer, &code, &message, &pass_message);
     goto FAILED;
     }
@@ -4230,13 +4326,12 @@ DEBUG(D_transport)
 
 if (sx->completed_addr && sx->ok && sx->send_quit)
   {
-  smtp_compare_t t_compare;
-
-  t_compare.tblock = tblock;
-  t_compare.current_sender_address = sender_address;
+  smtp_compare_t t_compare =
+    {.tblock = tblock, .current_sender_address = sender_address};
 
   if (  sx->first_addr			/* more addrs for this message */
-     || f.continue_more			/* more addrs for coninued-host */
+     || f.continue_more			/* more addrs for continued-host */
+     || tcw_done && tcw			/* more messages for host */
      || (
 #ifndef DISABLE_TLS
 	   (  tls_out.active.sock < 0  &&  !continue_proxy_cipher
@@ -4281,7 +4376,6 @@ if (sx->completed_addr && sx->ok && sx->send_quit)
 #endif
       int socket_fd = sx->cctx.sock;
 
-
       if (sx->first_addr)		/* More addresses still to be sent */
         {				/*   for this message              */
         continue_sequence++;		/* Causes * in logging */
@@ -4306,6 +4400,7 @@ if (sx->completed_addr && sx->ok && sx->send_quit)
 	  the socket on. */
 
 	  tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_WAIT);
+	  sx->send_tlsclose = FALSE;
 	  sx->cctx.tls_ctx = NULL;
 	  tls_out.active.sock = -1;
 	  smtp_peer_options = smtp_peer_options_wrap;
@@ -4399,20 +4494,21 @@ been passed to another process. */
 
 SEND_QUIT:
 if (sx->send_quit)
-			/* Use _MORE to get QUIT in FIN segment */
+  {			/* Use _MORE to get QUIT in FIN segment */
   (void)smtp_write_command(sx, SCMD_MORE, "QUIT\r\n");
-
-END_OFF:
-
 #ifndef DISABLE_TLS
-# ifdef EXIM_TCP_CORK
-if (sx->cctx.tls_ctx)	/* Use _CORK to get TLS Close Notify in FIN segment */
-  (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on));
+  if (sx->cctx.tls_ctx)
+    {
+# ifdef EXIM_TCP_CORK	/* Use _CORK to get TLS Close Notify in FIN segment */
+    (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on));
 # endif
-
-tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_NOWAIT);
-sx->cctx.tls_ctx = NULL;
+    tls_shutdown_wr(sx->cctx.tls_ctx);
+    sx->send_tlsclose = FALSE;
+    }
 #endif
+  }
+
+END_OFF:
 
 /* Close the socket, and return the appropriate value, first setting
 works because the NULL setting is passed back to the calling process, and
@@ -4424,26 +4520,58 @@ writing RSET might have failed, or there may be other addresses whose hosts are
 specified in the transports, and therefore not visible at top level, in which
 case continue_more won't get set. */
 
-HDEBUG(D_transport|D_acl|D_v) debug_printf_indent("  SMTP(close)>>\n");
 if (sx->send_quit)
   {
   /* This flushes data queued in the socket, being the QUIT and any TLS Close,
   sending them along with the client FIN flag.  Us (we hope) sending FIN first
   means we (client) take the TIME_WAIT state, so the server (which likely has a
-  higher connection rate) does no have to. */
+  higher connection rate) does not have to. */
 
+  HDEBUG(D_transport|D_acl|D_v) debug_printf_indent("  SMTP(shutdown)>>\n");
   shutdown(sx->cctx.sock, SHUT_WR);
+  }
 
+if (sx->send_quit || tcw_done && !tcw)
+  {
   /* Wait for (we hope) ack of our QUIT, and a server FIN.  Discard any data
   received, then discard the socket.  Any packet received after then, or receive
   data still in the socket, will get a RST - hence the pause/drain. */
 
+  /* Reap the response to QUIT, timing out after one second */
+  (void) smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), '2', 1);
+#ifndef DISABLE_TLS
+  if (sx->cctx.tls_ctx)
+    {
+    int n;
+
+    /* Reap the TLS Close Notify from the server, timing out after one second */
+    sigalrm_seen = FALSE;
+    ALARM(1);
+    do
+      n = tls_read(sx->cctx.tls_ctx, sx->inbuffer, sizeof(sx->inbuffer));
+    while (!sigalrm_seen && n > 0);
+    ALARM_CLR(0);
+
+# ifdef EXIM_TCP_CORK
+    (void) setsockopt(sx->cctx.sock, IPPROTO_TCP, EXIM_TCP_CORK, US &on, sizeof(on));
+# endif
+    tls_close(sx->cctx.tls_ctx, TLS_SHUTDOWN_WAIT);
+    sx->cctx.tls_ctx = NULL;
+    }
+#endif
   millisleep(20);
-  testharness_pause_ms(200);
   if (fcntl(sx->cctx.sock, F_SETFL, O_NONBLOCK) == 0)
-    for (int i = 16; read(sx->cctx.sock, sx->inbuffer, sizeof(sx->inbuffer)) > 0 && i > 0;)
-      i--;				/* drain socket */
+    for (int i = 16, n;						/* drain socket */
+	 (n = read(sx->cctx.sock, sx->inbuffer, sizeof(sx->inbuffer))) > 0 && i > 0;
+	 i--) HDEBUG(D_transport|D_acl|D_v)
+      {
+      int m = MIN(n, 64);
+      debug_printf_indent("  SMTP(drain %d bytes)<< %.*s\n", n, m, sx->inbuffer);
+      for (m = 0; m < n; m++)
+	debug_printf("0x%02x\n", sx->inbuffer[m]);
+      }
   }
+HDEBUG(D_transport|D_acl|D_v) debug_printf_indent("  SMTP(close)>>\n");
 (void)close(sx->cctx.sock);
 
 #ifndef DISABLE_EVENT
diff --git a/src/src/transports/smtp.h b/src/src/transports/smtp.h
index d55dc5961..1ea2a4d00 100644
--- a/src/src/transports/smtp.h
+++ b/src/src/transports/smtp.h
@@ -174,6 +174,7 @@ typedef struct {
   BOOL completed_addr:1;
   BOOL send_rset:1;
   BOOL send_quit:1;
+  BOOL send_tlsclose:1;
 
   int		max_rcpt;
   int		cmd_count;