Fix utf8clean not replacing incomplete final character

Before, it was just dropped, but we document that it's replaced by ?.

Tests updated, manual test-case for -be prompt is:

    ${utf8clean:${length_1:フィル}}

5 files changed, 36 insertions, 19 deletions

diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog
index bf5afe0af..bc521a6a3 100644
--- a/doc/doc-txt/ChangeLog
+++ b/doc/doc-txt/ChangeLog
@@ -92,6 +92,10 @@ JH/20 Bug 2296: Fix cutthrough for >1 address redirection.  Previously only
       time.  Either a crash (after delivery) or bogus log data could result.
       Discovery and analysis by Tim Stewart.
 
+PP/03 Make ${utf8clean:} expansion operator detect incomplete final character.
+      Previously if the string ended mid-character, we did not insert the
+      promised '?' replacement.
+
 
 Exim version 4.91
 -----------------
diff --git a/src/src/expand.c b/src/src/expand.c
index b6ff96aee..1508dcfd8 100644
--- a/src/src/expand.c
+++ b/src/src/expand.c
@@ -1683,7 +1683,7 @@ else if (host_lookup_deferred)
   g = string_catn(g, US";\n\tiprev=temperror", 19);
 else if (host_lookup_failed)
   g = string_catn(g, US";\n\tiprev=fail", 13);
-else 
+else
   return g;
 
 if (sender_host_address)
@@ -7143,12 +7143,13 @@ while (*s != 0)
         {
         int seq_len = 0, index = 0;
         int bytes_left = 0;
-	long codepoint = -1;
+        long codepoint = -1;
+        int complete;
         uschar seq_buff[4];			/* accumulate utf-8 here */
 
         while (*sub != 0)
 	  {
-	  int complete = 0;
+	  complete = 0;
 	  uschar c = *sub++;
 
 	  if (bytes_left)
@@ -7213,6 +7214,13 @@ while (*s != 0)
 			/* ASCII character follows incomplete sequence */
 	      yield = string_catn(yield, &c, 1);
 	  }
+        /* If given a sequence truncated mid-character, we also want to report ?
+        * Eg, ${length_1:フィル} is one byte, not one character, so we expect
+        * ${utf8clean:${length_1:フィル}} to yield '?' */
+        if (bytes_left != 0)
+          {
+          yield = string_catn(yield, UTF8_REPLACEMENT_CHAR, 1);
+          }
         continue;
         }
 
@@ -7954,7 +7962,7 @@ expand_hide_passwords(uschar * s)
 {
 return (  (  Ustrstr(s, "failed to expand") != NULL
 	  || Ustrstr(s, "expansion of ")    != NULL
-	  ) 
+	  )
        && (  Ustrstr(s, "mysql")   != NULL
 	  || Ustrstr(s, "pgsql")   != NULL
 	  || Ustrstr(s, "redis")   != NULL
@@ -7964,7 +7972,7 @@ return (  (  Ustrstr(s, "failed to expand") != NULL
 	  || Ustrstr(s, "ldapi:")  != NULL
 	  || Ustrstr(s, "ldapdn:") != NULL
 	  || Ustrstr(s, "ldapm:")  != NULL
-       )  ) 
+       )  )
   ? US"Temporary internal error" : s;
 }
 
diff --git a/test/confs/0600 b/test/confs/0600
index 29bc5bf1a..f6405460d 100644
--- a/test/confs/0600
+++ b/test/confs/0600
@@ -20,15 +20,17 @@ begin acl
 
 check_data:
   accept logwrite = \
-  			x-test-header-good1: ${utf8clean:$h_x-test-header-good1:}
- 		 logwrite = \
-  			x-test-header-good2: ${utf8clean:$h_x-test-header-good2:}
- 		 logwrite = \
-  			x-test-header-too-short: ${utf8clean:$h_x-test-header-too-short:}
- 		 logwrite = \
-  			x-test-header-too-long: ${utf8clean:$h_x-test-header-too-long:}
- 		 logwrite = \
-  			x-test-header-too-big: ${utf8clean:$h_x-test-header-too-big:}
+			x-test-header-good1: ${utf8clean:$h_x-test-header-good1:}
+		 logwrite = \
+			x-test-header-good2: ${utf8clean:$h_x-test-header-good2:}
+		 logwrite = \
+			x-test-header-too-short: ${utf8clean:$h_x-test-header-too-short:}
+		 logwrite = \
+			x-test-header-too-long: ${utf8clean:$h_x-test-header-too-long:}
+		 logwrite = \
+			x-test-header-too-big: ${utf8clean:$h_x-test-header-too-big:}
+		 log_write = \
+			x-test-header-truncated: ${utf8clean:$h_x-test-header-truncated:}
 
 
 
diff --git a/test/log/0600 b/test/log/0600
index 8fc8cfc36..a18b159d2 100644
--- a/test/log/0600
+++ b/test/log/0600
@@ -3,14 +3,16 @@
 1999-03-02 09:44:33 10HmaX-0005vi-00 x-test-header-too-short: ?.?.?.\303\244-?.-\303\234.?..?.-?.-?..-?.-?.-?.-?.-?..-?..?.
 1999-03-02 09:44:33 10HmaX-0005vi-00 x-test-header-too-long: ?????-\303\244-?????--\303\226-\303\204-\302\264-\340\244\221-\340\244\225-\340\244\234-\341\220\201-\341\221\214-\341\221\225-?????\360\253\237\206
 1999-03-02 09:44:33 10HmaX-0005vi-00 x-test-header-too-big: ?-----\363\200\200\200
+1999-03-02 09:44:33 10HmaX-0005vi-00 x-test-header-truncated: ?
 1999-03-02 09:44:33 10HmaX-0005vi-00 <= CALLER@the.local.host.name U=CALLER P=local-smtp S=sss
 1999-03-02 09:44:33 10HmaX-0005vi-00 => CALLER <CALLER@the.local.host.name> R=localuser T=local_delivery
 1999-03-02 09:44:33 10HmaX-0005vi-00 Completed
-1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-good1: 
-1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-good2: 
-1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-too-short: 
-1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-too-long: 
-1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-too-big: 
+1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-good1:
+1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-good2:
+1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-too-short:
+1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-too-long:
+1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-too-big:
+1999-03-02 09:44:33 10HmaY-0005vi-00 x-test-header-too-truncated:
 1999-03-02 09:44:33 10HmaY-0005vi-00 <= CALLER@the.local.host.name U=CALLER P=local-smtp S=sss
 1999-03-02 09:44:33 10HmaY-0005vi-00 => CALLER <CALLER@the.local.host.name> R=localuser T=local_delivery
 1999-03-02 09:44:33 10HmaY-0005vi-00 Completed
diff --git a/test/scripts/0000-Basic/0600 b/test/scripts/0000-Basic/0600
index 9d5e67b5b..43f728822 100644
--- a/test/scripts/0000-Basic/0600
+++ b/test/scripts/0000-Basic/0600
@@ -10,6 +10,7 @@ x-test-header-good2: ßüöä€ÜÖÄ´ऑकजᐁᑌᑕ𫝆𫟘
 x-test-header-too-short: �.�.�.ä-�.-Ü.�..�.-�.-�..-�.-�.-�.-�.-�..-�..�.
 x-test-header-too-long: �����-ä-�����--Ö-Ä-´-ऑ-क-ज-ᐁ-ᑌ-ᑕ-�����𫟆
 x-test-header-too-big: ����-----󀀀
+x-test-header-truncated: �
 Subject: This is a test message.
 
 This is a test message.