I18N: add a utf8_downconvert option to the smtp transport. Bug 2248

12 files changed, 110 insertions, 15 deletions

diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt
index d0e3358b8..fb5944d95 100644
--- a/doc/doc-docbook/spec.xfpt
+++ b/doc/doc-docbook/spec.xfpt
@@ -24716,6 +24716,16 @@ The &%tls_verify_certificates%& option must also be set.
 If both this option and &%tls_try_verify_hosts%& are unset
 operation is as if this option selected all hosts.
 
+.new
+.option utf8_downconvert smtp integer!! unset
+.cindex utf8 "address downconversion"
+.cindex i18n "utf8 address downconversion"
+If built with internationalization support,
+this option controls conversion of UTF-8 in message addresses
+to a-label form.
+For details see section &<<SECTi18nMTA>>&.
+.wen
+
 
 
 
@@ -39863,6 +39873,12 @@ If a value is appended it may be:
 If mua_wrapper is set, the utf8_downconvert control
 is initially set to -1.
 
+.new
+The smtp transport has an option &%utf8_downconvert%&.
+If set it must expand to one of the three values described above,
+and it overrides any previously set value.
+.wen
+
 
 There is no explicit support for VRFY and EXPN.
 Configurations supporting these should inspect
diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff
index 4039895b6..05c416ec7 100644
--- a/doc/doc-txt/NewStuff
+++ b/doc/doc-txt/NewStuff
@@ -19,6 +19,8 @@ Version 4.92
 
  3. EXPERIMENTAL_REQUIRETLS.  See the experimental.spec file.
 
+ 4. If built with SUPPORT_I18N a "utf8_downconvert" option on the smtp transport.
+
 Version 4.91
 --------------
 
diff --git a/doc/doc-txt/OptionLists.txt b/doc/doc-txt/OptionLists.txt
index 05e47c0e4..c3ce62d45 100644
--- a/doc/doc-txt/OptionLists.txt
+++ b/doc/doc-txt/OptionLists.txt
@@ -613,6 +613,7 @@ use_mbx_lock                         boolean         +             appendfile
 use_shell                            boolean         false         pipe              1.70
 user                                 string          +             routers           4.00
                                                      unset         transports        4.00 replaces individual options
+utf8_downcvt			     integer	     unset	   smtp		     4.92 if SUPPORT_I18N
 uucp_from_pattern                    string          +             main              1.75
 uucp_from_sender                     string*         "$1"          main              1.75
 verify                               boolean         true          routers           4.00
diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index 08d1810d6..b95913c59 100644
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -190,15 +190,18 @@ optionlist smtp_transport_options[] = {
   { "tls_verify_certificates", opt_stringptr,
       (void *)offsetof(smtp_transport_options_block, tls_verify_certificates) },
   { "tls_verify_hosts",     opt_stringptr,
-      (void *)offsetof(smtp_transport_options_block, tls_verify_hosts) }
+      (void *)offsetof(smtp_transport_options_block, tls_verify_hosts) },
+#endif
+#ifdef SUPPORT_I18N
+  { "utf8_downconvert",	    opt_stringptr,
+      (void *)offsetof(smtp_transport_options_block, utf8_downconvert) },
 #endif
 };
 
 /* Size of the options list. An extern variable has to be used so that its
 address can appear in the tables drtables.c. */
 
-int smtp_transport_options_count =
-  sizeof(smtp_transport_options)/sizeof(optionlist);
+int smtp_transport_options_count = nelem(smtp_transport_options);
 
 
 #ifdef MACRO_PREDEF
@@ -287,6 +290,9 @@ smtp_transport_options_block smtp_transport_option_defaults = {
   .tls_try_verify_hosts =	US"*",
   .tls_verify_cert_hostnames =	US"*",
 #endif
+#ifdef SUPPORT_I18N
+  .utf8_downconvert =		NULL,
+#endif
 #ifndef DISABLE_DKIM
  .dkim =
    {.dkim_domain =		NULL,
@@ -2218,6 +2224,38 @@ sx->setting_up = FALSE;
 #ifdef SUPPORT_I18N
 if (sx->addrlist->prop.utf8_msg)
   {
+  uschar * s;
+
+  /* If the transport sets a downconversion mode it overrides any set by ACL
+  for the message. */
+
+  if ((s = sx->ob->utf8_downconvert))
+    {
+    if (!(s = expand_string(s)))
+      {
+      message = string_sprintf("failed to expand utf8_downconvert: %s",
+        expand_string_message);
+      set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE);
+      yield = DEFER;
+      goto SEND_QUIT;
+      }
+    switch (*s)
+      {
+      case '1':	sx->addrlist->prop.utf8_downcvt = TRUE;
+		sx->addrlist->prop.utf8_downcvt_maybe = FALSE;
+		break;
+      case '0':	sx->addrlist->prop.utf8_downcvt = FALSE;
+		sx->addrlist->prop.utf8_downcvt_maybe = FALSE;
+		break;
+      case '-':	if (s[1] == '1')
+		  {
+		  sx->addrlist->prop.utf8_downcvt = FALSE;
+		  sx->addrlist->prop.utf8_downcvt_maybe = TRUE;
+		  }
+		break;
+      }
+    }
+
   sx->utf8_needed = !sx->addrlist->prop.utf8_downcvt
 		    && !sx->addrlist->prop.utf8_downcvt_maybe;
   DEBUG(D_transport) if (!sx->utf8_needed)
@@ -2231,7 +2269,7 @@ if (sx->utf8_needed && !(sx->peer_offered & OPTION_UTF8))
   errno = ERRNO_UTF8_FWD;
   goto RESPONSE_FAILED;
   }
-#endif
+#endif	/*SUPPORT_I18N*/
 
 #if defined(SUPPORT_TLS) && defined(EXPERIMENTAL_REQUIRETLS)
   /*XXX should tls_requiretls actually be per-addr? */
diff --git a/src/src/transports/smtp.h b/src/src/transports/smtp.h
index 34c49d930..df9644377 100644
--- a/src/src/transports/smtp.h
+++ b/src/src/transports/smtp.h
@@ -88,6 +88,9 @@ typedef struct {
   uschar *tls_try_verify_hosts;
   uschar *tls_verify_cert_hostnames;
 #endif
+#ifdef SUPPORT_I18N
+  uschar *utf8_downconvert;
+#endif
 #ifndef DISABLE_DKIM
   struct ob_dkim dkim;
 #endif
diff --git a/test/confs/4201 b/test/confs/4201
index 9242eb063..17afe9f78 100644
--- a/test/confs/4201
+++ b/test/confs/4201
@@ -124,5 +124,8 @@ local_delivery:
 
 rmt_smtp:
   driver = smtp
+.ifdef STRICT
+  utf8_downconvert = STRICT
+.endif
 
 # End
diff --git a/test/log/4207 b/test/log/4207
index e39c0f5fb..3bd01a3dd 100644
--- a/test/log/4207
+++ b/test/log/4207
@@ -1,17 +1,23 @@
-1999-03-02 09:44:33 10HmaX-0005vi-00 <= 他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com U=CALLER P=utf8local-esmtp S=sss for user.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex
-1999-03-02 09:44:33 10HmaX-0005vi-00 => xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@test.ex <user.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex> F=<xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com> R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] C="250 OK id=10HmaY-0005vi-00"
+1999-03-02 09:44:33 10HmaX-0005vi-00 <= 他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com U=CALLER P=utf8local-esmtp S=sss for user1.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex
+1999-03-02 09:44:33 10HmaX-0005vi-00 => xn--user1.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex <user1.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex> F=<xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com> R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] C="250 OK id=10HmaY-0005vi-00"
 1999-03-02 09:44:33 10HmaX-0005vi-00 Completed
-1999-03-02 09:44:33 10HmaZ-0005vi-00 <= 他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com U=CALLER P=utf8local-esmtp S=sss for zuser.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex
-1999-03-02 09:44:33 10HmaZ-0005vi-00 => xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@test.ex <zuser.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex> F=<xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com> R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] C="250 OK id=10HmbA-0005vi-00"
+1999-03-02 09:44:33 10HmaZ-0005vi-00 <= 他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com U=CALLER P=utf8local-esmtp S=sss for zuser2.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex
+1999-03-02 09:44:33 10HmaZ-0005vi-00 => xn--user2.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex <zuser2.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex> F=<xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com> R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] C="250 OK id=10HmbA-0005vi-00"
 1999-03-02 09:44:33 10HmaZ-0005vi-00 Completed
+1999-03-02 09:44:33 10HmbB-0005vi-00 <= 他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com U=CALLER P=utf8local-esmtp S=sss for user3.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex
+1999-03-02 09:44:33 10HmbB-0005vi-00 => xn--user3.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex <user3.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex> F=<xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com> R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] C="250 OK id=10HmbC-0005vi-00"
+1999-03-02 09:44:33 10HmbB-0005vi-00 Completed
 
 ******** SERVER ********
 1999-03-02 09:44:33 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTP on port 1225
-1999-03-02 09:44:33 10HmaY-0005vi-00 <= xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com H=localhost (the.local.host.name) [127.0.0.1] P=esmtp S=sss id=E10HmaX-0005vi-00@the.local.host.name for xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@test.ex
-1999-03-02 09:44:33 10HmbA-0005vi-00 <= xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com H=localhost (the.local.host.name) [127.0.0.1] P=esmtp S=sss id=E10HmaZ-0005vi-00@the.local.host.name for xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@test.ex
+1999-03-02 09:44:33 10HmaY-0005vi-00 <= xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com H=localhost (the.local.host.name) [127.0.0.1] P=esmtp S=sss id=E10HmaX-0005vi-00@the.local.host.name for xn--user1.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex
+1999-03-02 09:44:33 10HmbA-0005vi-00 <= xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com H=localhost (the.local.host.name) [127.0.0.1] P=esmtp S=sss id=E10HmaZ-0005vi-00@the.local.host.name for xn--user2.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex
+1999-03-02 09:44:33 10HmbC-0005vi-00 <= xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com H=localhost (the.local.host.name) [127.0.0.1] P=esmtp S=sss id=E10HmbB-0005vi-00@the.local.host.name for xn--user3.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex
 1999-03-02 09:44:33 Start queue run: pid=pppp -qqff
-1999-03-02 09:44:33 10HmaY-0005vi-00 => :blackhole: <xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@test.ex> R=localuser
+1999-03-02 09:44:33 10HmaY-0005vi-00 => :blackhole: <xn--user1.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex> R=localuser
 1999-03-02 09:44:33 10HmaY-0005vi-00 Completed
-1999-03-02 09:44:33 10HmbA-0005vi-00 => :blackhole: <xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@test.ex> R=localuser
+1999-03-02 09:44:33 10HmbA-0005vi-00 => :blackhole: <xn--user2.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex> R=localuser
 1999-03-02 09:44:33 10HmbA-0005vi-00 Completed
+1999-03-02 09:44:33 10HmbC-0005vi-00 => :blackhole: <xn--user3.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex> R=localuser
+1999-03-02 09:44:33 10HmbC-0005vi-00 Completed
 1999-03-02 09:44:33 End queue run: pid=pppp -qqff
diff --git a/test/runtest b/test/runtest
index 9f556660d..a38f112af 100755
--- a/test/runtest
+++ b/test/runtest
@@ -1613,6 +1613,7 @@ $munges =
                   |hosts_(avoid|nopass|noproxy|require|verify_avoid)_tls
                   |socks_proxy
                   |tls_[^ ]*
+		  |utf8_downconvert
                   )($|[ ]=)/x'
     },
 
diff --git a/test/scripts/0000-Basic/0577 b/test/scripts/0000-Basic/0577
index 1bd510ae3..bbd2954c7 100644
--- a/test/scripts/0000-Basic/0577
+++ b/test/scripts/0000-Basic/0577
@@ -1,2 +1,3 @@
 # Test different variants of .includes
+munge optional_config
 exim -bP config
diff --git a/test/scripts/4200-International/4207 b/test/scripts/4200-International/4207
index 061a2c6b3..fa32bec92 100644
--- a/test/scripts/4200-International/4207
+++ b/test/scripts/4200-International/4207
@@ -9,7 +9,7 @@ exim -DSERVER=server -DOPTION="" -bd -oX PORT_D
 exim -bs -odi -DCONTROL="control=utf8_downconvert"
 EHLO client.bh
 MAIL FROM: <他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com> SMTPUTF8
-RCPT TO: <user.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex>
+RCPT TO: <user1.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex>
 DATA
 Subject: test
 
@@ -22,7 +22,20 @@ QUIT
 exim -bs -odi -DCONTROL="control=utf8_downconvert/-1"
 EHLO client.bh
 MAIL FROM: <他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com> SMTPUTF8
-RCPT TO: <zuser.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex>
+RCPT TO: <zuser2.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex>
+DATA
+Subject: test
+
+body
+.
+QUIT
+****
+#
+# utf-8 from, never-downconvert set by ACL, overridden by transport
+exim -bs -odi -DCONTROL="control=utf8_downconvert/0" -DSTRICT=1
+EHLO client.bh
+MAIL FROM: <他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com> SMTPUTF8
+RCPT TO: <user3.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex>
 DATA
 Subject: test
 
diff --git a/test/stdout/0577 b/test/stdout/0577
index 80cf3da88..ef918a86b 100644
--- a/test/stdout/0577
+++ b/test/stdout/0577
@@ -14,7 +14,6 @@ gecos_name = CALLER_NAME
 dns_cname_loops = 9
 chunking_advertise_hosts =
 # 1 "TESTSUITE/aux-var/std_conf_prefix"
-tls_advertise_hosts =
 # 1 "TESTSUITE/test-config"
 # 2 "TESTSUITE/test-config"
 # 1 "TESTSUITE/confs/0577./aaa"
diff --git a/test/stdout/4207 b/test/stdout/4207
index 948434ff3..34a2cfe36 100644
--- a/test/stdout/4207
+++ b/test/stdout/4207
@@ -22,3 +22,15 @@
 354 Enter message, ending with "." on a line by itself

 250 OK id=10HmaZ-0005vi-00

 221 the.local.host.name closing connection

+220 the.local.host.name ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000

+250-the.local.host.name Hello CALLER at client.bh

+250-SIZE 52428800

+250-8BITMIME

+250-PIPELINING

+250-SMTPUTF8

+250 HELP

+250 OK

+250 Accepted

+354 Enter message, ending with "." on a line by itself

+250 OK id=10HmbB-0005vi-00

+221 the.local.host.name closing connection