Fix crash in transport, on second smtp-connect fail for a list of target hosts

Reported as the sequence:
  1MX: 554 on connect (banner)
  2MX: TCP conn timeout

12 files changed, 78 insertions, 11 deletions

diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog
index 4eb0d3b1a..7f2e9d097 100644
--- a/doc/doc-txt/ChangeLog
+++ b/doc/doc-txt/ChangeLog
@@ -102,6 +102,10 @@ JH/14 For the "sock" variant of the malware scanner interface, accept an empty
       cmdline element to get the documented default one.  Previously it was
       inaccessible.
 
+JH/15 Fix a crash in the smtp transport caused when two hosts in succession
+      are unsuable for non-message-specific reasons - eg. connection timeout,
+      banner-time rejection.
+
 
 Exim version 4.89
 -----------------
diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index 454c0f74d..dc9e03be1 100644
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -2177,25 +2177,34 @@ return OK;
 
   /* The failure happened while setting up the call; see if the failure was
   a 5xx response (this will either be on connection, or following HELO - a 5xx
-  after EHLO causes it to try HELO). If so, fail all addresses, as this host is
-  never going to accept them. For other errors during setting up (timeouts or
-  whatever), defer all addresses, and yield DEFER, so that the host is not
-  tried again for a while. */
+  after EHLO causes it to try HELO). If so, and there are no more hosts to try,
+  fail all addresses, as this host is never going to accept them. For other
+  errors during setting up (timeouts or whatever), defer all addresses, and
+  yield DEFER, so that the host is not tried again for a while.
+
+  XXX This peeking for another host feels like a layering violation. We want
+  to note the host as unusable, but down here we shouldn't know if this was
+  the last host to try for the addr(list).  Perhaps the upper layer should be
+  the one to do set_errno() ?  The problem is that currently the addr is where
+  errno etc. are stashed, but until we run out of hosts to try the errors are
+  host-specific.  Maybe we should enhance the host_item definition? */
 
 FAILED:
   sx->ok = FALSE;                /* For when reached by GOTO */
-
-  yield = code == '5'
+  set_errno(sx->addrlist, errno, message,
+	    sx->host->next
+	    ? DEFER
+	    : code == '5'
 #ifdef SUPPORT_I18N
-	  || errno == ERRNO_UTF8_FWD
+			|| errno == ERRNO_UTF8_FWD
 #endif
-    ? FAIL : DEFER;
-
-  set_errno(sx->addrlist, errno, message, yield, pass_message, sx->host
+	    ? FAIL : DEFER,
+	    pass_message, sx->host
 #ifdef EXPERIMENTAL_DSN_INFO
 	    , sx->smtp_greeting, sx->helo_response
 #endif
 	    );
+  yield = DEFER;
   }
 
 
diff --git a/test/confs/0211 b/test/confs/0211
index 96f3beac1..7856481df 100644
--- a/test/confs/0211
+++ b/test/confs/0211
@@ -30,7 +30,11 @@ begin routers
 others:
   driver = manualroute
   domains = ! +local_domains
-  route_list = * localhost4.test.ex byname
+.ifdef LIST
+  route_data = 127.0.0.1:HOSTIPV4
+.else
+  route_data = localhost4.test.ex byname
+.endif
   self = send
   transport = smtp
   no_more
@@ -48,6 +52,7 @@ begin transports
 smtp:
   driver = smtp
   port = PORT_S
+  command_timeout = 1s
 
 local_delivery:
   driver = appendfile
diff --git a/test/log/0211 b/test/log/0211
index 74acec3a3..ac647a67b 100644
--- a/test/log/0211
+++ b/test/log/0211
@@ -14,6 +14,7 @@
 1999-03-02 09:44:33 End queue run: pid=pppp -qf
 1999-03-02 09:44:33 Test: reject connect
 1999-03-02 09:44:33 Start queue run: pid=pppp -qf
+1999-03-02 09:44:33 10HmaX-0005vi-00 H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after initial connection: 550 Go away (A)
 1999-03-02 09:44:33 10HmaX-0005vi-00 ** userx@domain1 F=<CALLER@test.ex> R=others T=smtp H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after initial connection: 550 Go away (A)
 1999-03-02 09:44:33 10HmaY-0005vi-00 <= <> R=10HmaX-0005vi-00 U=EXIMUSER P=local S=sss
 1999-03-02 09:44:33 10HmaY-0005vi-00 => CALLER <CALLER@test.ex> F=<> R=all T=local_delivery
@@ -22,6 +23,7 @@
 1999-03-02 09:44:33 End queue run: pid=pppp -qf
 1999-03-02 09:44:33 Test: reject helo
 1999-03-02 09:44:33 10HmaZ-0005vi-00 <= CALLER@test.ex U=CALLER P=local S=sss
+1999-03-02 09:44:33 10HmaZ-0005vi-00 H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after HELO the.local.host.name: 550 Go away (C)
 1999-03-02 09:44:33 10HmaZ-0005vi-00 ** userx@domain1 F=<CALLER@test.ex> R=others T=smtp H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after HELO the.local.host.name: 550 Go away (C)
 1999-03-02 09:44:33 10HmaZ-0005vi-00 ** usery@domain2 F=<CALLER@test.ex> R=others T=smtp H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after HELO the.local.host.name: 550 Go away (C)
 1999-03-02 09:44:33 10HmbA-0005vi-00 <= <> R=10HmaZ-0005vi-00 U=EXIMUSER P=local S=sss
@@ -32,3 +34,9 @@
 1999-03-02 09:44:33 10HmbB-0005vi-00 <= CALLER@test.ex U=CALLER P=local S=sss
 1999-03-02 09:44:33 10HmbB-0005vi-00 H=localhost4.test.ex [127.0.0.1]: Remote host closed connection in response to HELO the.local.host.name (EHLO response was: 550 You are banned)
 1999-03-02 09:44:33 10HmbB-0005vi-00 == userx@domain1 R=others T=smtp defer (-18) H=localhost4.test.ex [127.0.0.1]: Remote host closed connection in response to HELO the.local.host.name (EHLO response was: 550 You are banned)
+1999-03-02 09:44:33 Test: smtp-reject conn on 1MX, timeout TCP conn on 2MX
+1999-03-02 09:44:33 Start queue run: pid=pppp -qf
+1999-03-02 09:44:33 10HmbB-0005vi-00 H=127.0.0.1 [127.0.0.1]: SMTP error from remote mail server after initial connection: 554 no smtp service here
+1999-03-02 09:44:33 10HmbB-0005vi-00 H=ip4.ip4.ip4.ip4 [ip4.ip4.ip4.ip4]: SMTP timeout after initial connection: Connection timed out
+1999-03-02 09:44:33 10HmbB-0005vi-00 == userx@domain1 R=others T=smtp defer (dd): Connection timed out H=ip4.ip4.ip4.ip4 [ip4.ip4.ip4.ip4]: SMTP timeout after initial connection
+1999-03-02 09:44:33 End queue run: pid=pppp -qf
diff --git a/test/log/0429 b/test/log/0429
index cb75f6cf4..d477a6b9b 100644
--- a/test/log/0429
+++ b/test/log/0429
@@ -1,5 +1,6 @@
 1999-03-02 09:44:33 10HmaX-0005vi-00 <= CALLER@myhost.test.ex U=CALLER P=local S=sss
 1999-03-02 09:44:33 Start queue run: pid=pppp
+1999-03-02 09:44:33 10HmaX-0005vi-00 H=127.0.0.1 [127.0.0.1]: SMTP error from remote mail server after HELO myhost.test.ex: 550 No
 1999-03-02 09:44:33 10HmaX-0005vi-00 ** x@y R=r1 T=t1 H=127.0.0.1 [127.0.0.1]: SMTP error from remote mail server after HELO myhost.test.ex: 550 No
 1999-03-02 09:44:33 10HmaY-0005vi-00 <= <> R=10HmaX-0005vi-00 U=EXIMUSER P=local S=sss
 1999-03-02 09:44:33 10HmaX-0005vi-00 Completed
diff --git a/test/log/4203 b/test/log/4203
index a1acf570e..1c2d52830 100644
--- a/test/log/4203
+++ b/test/log/4203
@@ -1,4 +1,5 @@
 1999-03-02 09:44:33 10HmaX-0005vi-00 <= यहलोगहिन्दीक्योंनहींबोलसकतेहैं@japanese.なぜみんな日本語を話してくれないのか.local U=CALLER P=utf8local-esmtp S=sss for userz@test.ex
+1999-03-02 09:44:33 10HmaX-0005vi-00 H=127.0.0.1 [127.0.0.1]: utf8 support required but not offered for forwarding
 1999-03-02 09:44:33 10HmaX-0005vi-00 ** userz@test.ex F=<यहलोगहिन्दीक्योंनहींबोलसकतेहैं@japanese.なぜみんな日本語を話してくれないのか.local> R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1]: utf8 support required but not offered for forwarding
 1999-03-02 09:44:33 10HmaY-0005vi-00 <= <> R=10HmaX-0005vi-00 U=EXIMUSER P=local S=sss for यहलोगहिन्दीक्योंनहींबोलसकतेहैं@japanese.なぜみんな日本語を話してくれないのか.local
 1999-03-02 09:44:33 10HmaY-0005vi-00 no immediate delivery: queued by ACL
diff --git a/test/log/4213 b/test/log/4213
index e0c0a43d9..1a9aa5eae 100644
--- a/test/log/4213
+++ b/test/log/4213
@@ -1,4 +1,5 @@
 1999-03-02 09:44:33 10HmaX-0005vi-00 <= यहलोगहिन्दीक्योंनहींबोलसकतेहैं@japanese.なぜみんな日本語を話してくれないのか.local U=CALLER P=utf8local-esmtp S=sss for userz@test.ex
+1999-03-02 09:44:33 10HmaX-0005vi-00 H=127.0.0.1 [127.0.0.1]: utf8 support required but not offered for forwarding
 1999-03-02 09:44:33 10HmaX-0005vi-00 ** userz@test.ex F=<यहलोगहिन्दीक्योंनहींबोलसकतेहैं@japanese.なぜみんな日本語を話してくれないのか.local> R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] X=TLS1.x:xxxxRSA_AES_256_CBC_SHAnnn:256 CV=no: utf8 support required but not offered for forwarding
 1999-03-02 09:44:33 10HmaY-0005vi-00 <= <> R=10HmaX-0005vi-00 U=EXIMUSER P=local S=sss for यहलोगहिन्दीक्योंनहींबोलसकतेहैं@japanese.なぜみんな日本語を話してくれないのか.local
 1999-03-02 09:44:33 10HmaY-0005vi-00 no immediate delivery: queued by ACL
diff --git a/test/log/4223 b/test/log/4223
index fb5a1defe..f4d4efa3a 100644
--- a/test/log/4223
+++ b/test/log/4223
@@ -1,4 +1,5 @@
 1999-03-02 09:44:33 10HmaX-0005vi-00 <= यहलोगहिन्दीक्योंनहींबोलसकतेहैं@japanese.なぜみんな日本語を話してくれないのか.local U=CALLER P=utf8local-esmtp S=sss for userz@test.ex
+1999-03-02 09:44:33 10HmaX-0005vi-00 H=127.0.0.1 [127.0.0.1]: utf8 support required but not offered for forwarding
 1999-03-02 09:44:33 10HmaX-0005vi-00 ** userz@test.ex F=<यहलोगहिन्दीक्योंनहींबोलसकतेहैं@japanese.なぜみんな日本語を話してくれないのか.local> R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] X=TLSv1:AES256-SHA:256 CV=no: utf8 support required but not offered for forwarding
 1999-03-02 09:44:33 10HmaY-0005vi-00 <= <> R=10HmaX-0005vi-00 U=EXIMUSER P=local S=sss for यहलोगहिन्दीक्योंनहींबोलसकतेहैं@japanese.なぜみんな日本語を話してくれないのか.local
 1999-03-02 09:44:33 10HmaY-0005vi-00 no immediate delivery: queued by ACL
diff --git a/test/log/4700 b/test/log/4700
index 35da72e04..8a1a7a4f4 100644
--- a/test/log/4700
+++ b/test/log/4700
@@ -6,6 +6,7 @@
 1999-03-02 09:44:33 10HmaX-0005vi-00 == userx@domain1 R=others T=smtp defer (0) H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after HELO the.local.host.name: 450 I'm busy
 1999-03-02 09:44:33 End queue run: pid=pppp -qf
 1999-03-02 09:44:33 Start queue run: pid=pppp -qf
+1999-03-02 09:44:33 10HmaX-0005vi-00 H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after initial connection: 550 Go away
 1999-03-02 09:44:33 10HmaX-0005vi-00 ** userx@domain1 F=<CALLER@test.ex> R=others T=smtp H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after initial connection: 550 Go away
 1999-03-02 09:44:33 10HmaY-0005vi-00 <= <> R=10HmaX-0005vi-00 U=EXIMUSER P=local S=sss
 1999-03-02 09:44:33 10HmaY-0005vi-00 => CALLER <CALLER@test.ex> F=<> R=all T=local_delivery
@@ -13,6 +14,7 @@
 1999-03-02 09:44:33 10HmaX-0005vi-00 Completed
 1999-03-02 09:44:33 End queue run: pid=pppp -qf
 1999-03-02 09:44:33 10HmaZ-0005vi-00 <= CALLER@test.ex U=CALLER P=local S=sss
+1999-03-02 09:44:33 10HmaZ-0005vi-00 H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after HELO the.local.host.name: 550 Go away
 1999-03-02 09:44:33 10HmaZ-0005vi-00 ** userx@domain1 F=<CALLER@test.ex> R=others T=smtp H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after HELO the.local.host.name: 550 Go away
 1999-03-02 09:44:33 10HmaZ-0005vi-00 ** usery@domain2 F=<CALLER@test.ex> R=others T=smtp H=localhost4.test.ex [127.0.0.1]: SMTP error from remote mail server after HELO the.local.host.name: 550 Go away
 1999-03-02 09:44:33 10HmbA-0005vi-00 <= <> R=10HmaZ-0005vi-00 U=EXIMUSER P=local S=sss
diff --git a/test/scripts/0000-Basic/0211 b/test/scripts/0000-Basic/0211
index 655b4dd96..c27a7ebf3 100644
--- a/test/scripts/0000-Basic/0211
+++ b/test/scripts/0000-Basic/0211
@@ -13,6 +13,7 @@ exim -odi userx@domain1
 Test message 1
 ****
 #
+sudo rm DIR/spool/db/*
 exim -z "Test: temp-rej helo"
 ****
 server PORT_S
@@ -27,6 +28,7 @@ QUIT
 exim -qf
 ****
 #
+sudo rm DIR/spool/db/*
 exim -z "Test: drop conn after banner"
 ****
 server PORT_S
@@ -35,6 +37,7 @@ server PORT_S
 exim -qf
 ****
 #
+sudo rm DIR/spool/db/*
 exim -z "Test: reject connect"
 ****
 server PORT_S
@@ -45,6 +48,7 @@ QUIT
 exim -qf
 ****
 #
+sudo rm DIR/spool/db/*
 exim -z "Test: reject helo"
 ****
 server PORT_S
@@ -74,4 +78,19 @@ Test message 4
 ****
 #
 #
+#
+sudo rm DIR/spool/db/*
+exim -z "Test: smtp-reject conn on 1MX, timeout TCP conn on 2MX"
+****
+server PORT_S 2
+554 no smtp service here
+QUIT
+220 bye
+*eof
+*sleep 2
+****
+exim -DLIST -qf
+****
+#
+#
 no_msglog_check
diff --git a/test/scripts/4700-dsn-info/4700 b/test/scripts/4700-dsn-info/4700
index d0dd4f363..29b2ce8b4 100644
--- a/test/scripts/4700-dsn-info/4700
+++ b/test/scripts/4700-dsn-info/4700
@@ -9,6 +9,8 @@ QUIT
 exim -odi userx@domain1
 Test message 1
 ****
+sudo rm DIR/spool/db/*
+#
 server PORT_S
 220 Connected OK
 EHLO
@@ -20,6 +22,8 @@ QUIT
 ****
 exim -qf
 ****
+sudo rm DIR/spool/db/*
+#
 server PORT_S
 550 Go away
 QUIT
@@ -27,6 +31,8 @@ QUIT
 ****
 exim -qf
 ****
+sudo rm DIR/spool/db/*
+#
 server PORT_S
 220 Connected OK
 EHLO
diff --git a/test/stdout/0211 b/test/stdout/0211
index 2d484bda5..708632ded 100644
--- a/test/stdout/0211
+++ b/test/stdout/0211
@@ -43,3 +43,13 @@ EHLO the.local.host.name
 550 You are banned
 HELO the.local.host.name
 End of script
+Listening on port 1224 ... 
+Connection request from [127.0.0.1]
+554 no smtp service here
+QUIT
+220 bye
+Expected EOF read from client
+Listening on port 1224 ... 
+Connection request from [ip4.ip4.ip4.ip4]
+*sleep 2
+End of script