From 71c158466dab1d452d450843e8c204a42200b7a8 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sat, 28 Jul 2018 20:48:19 +0100 Subject: I18N: add a utf8_downconvert option to the smtp transport. Bug 2248 --- doc/doc-docbook/spec.xfpt | 16 +++++++++++++ doc/doc-txt/NewStuff | 2 ++ doc/doc-txt/OptionLists.txt | 1 + src/src/transports/smtp.c | 46 ++++++++++++++++++++++++++++++++---- src/src/transports/smtp.h | 3 +++ test/confs/4201 | 3 +++ test/log/4207 | 22 ++++++++++------- test/runtest | 1 + test/scripts/0000-Basic/0577 | 1 + test/scripts/4200-International/4207 | 17 +++++++++++-- test/stdout/0577 | 1 - test/stdout/4207 | 12 ++++++++++ 12 files changed, 110 insertions(+), 15 deletions(-) diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt index d0e3358b8..fb5944d95 100644 --- a/doc/doc-docbook/spec.xfpt +++ b/doc/doc-docbook/spec.xfpt @@ -24716,6 +24716,16 @@ The &%tls_verify_certificates%& option must also be set. If both this option and &%tls_try_verify_hosts%& are unset operation is as if this option selected all hosts. +.new +.option utf8_downconvert smtp integer!! unset +.cindex utf8 "address downconversion" +.cindex i18n "utf8 address downconversion" +If built with internationalization support, +this option controls conversion of UTF-8 in message addresses +to a-label form. +For details see section &<>&. +.wen + @@ -39863,6 +39873,12 @@ If a value is appended it may be: If mua_wrapper is set, the utf8_downconvert control is initially set to -1. +.new +The smtp transport has an option &%utf8_downconvert%&. +If set it must expand to one of the three values described above, +and it overrides any previously set value. +.wen + There is no explicit support for VRFY and EXPN. Configurations supporting these should inspect diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff index 4039895b6..05c416ec7 100644 --- a/doc/doc-txt/NewStuff +++ b/doc/doc-txt/NewStuff @@ -19,6 +19,8 @@ Version 4.92 3. EXPERIMENTAL_REQUIRETLS. See the experimental.spec file. + 4. If built with SUPPORT_I18N a "utf8_downconvert" option on the smtp transport. + Version 4.91 -------------- diff --git a/doc/doc-txt/OptionLists.txt b/doc/doc-txt/OptionLists.txt index 05e47c0e4..c3ce62d45 100644 --- a/doc/doc-txt/OptionLists.txt +++ b/doc/doc-txt/OptionLists.txt @@ -613,6 +613,7 @@ use_mbx_lock boolean + appendfile use_shell boolean false pipe 1.70 user string + routers 4.00 unset transports 4.00 replaces individual options +utf8_downcvt integer unset smtp 4.92 if SUPPORT_I18N uucp_from_pattern string + main 1.75 uucp_from_sender string* "$1" main 1.75 verify boolean true routers 4.00 diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c index 08d1810d6..b95913c59 100644 --- a/src/src/transports/smtp.c +++ b/src/src/transports/smtp.c @@ -190,15 +190,18 @@ optionlist smtp_transport_options[] = { { "tls_verify_certificates", opt_stringptr, (void *)offsetof(smtp_transport_options_block, tls_verify_certificates) }, { "tls_verify_hosts", opt_stringptr, - (void *)offsetof(smtp_transport_options_block, tls_verify_hosts) } + (void *)offsetof(smtp_transport_options_block, tls_verify_hosts) }, +#endif +#ifdef SUPPORT_I18N + { "utf8_downconvert", opt_stringptr, + (void *)offsetof(smtp_transport_options_block, utf8_downconvert) }, #endif }; /* Size of the options list. An extern variable has to be used so that its address can appear in the tables drtables.c. */ -int smtp_transport_options_count = - sizeof(smtp_transport_options)/sizeof(optionlist); +int smtp_transport_options_count = nelem(smtp_transport_options); #ifdef MACRO_PREDEF @@ -287,6 +290,9 @@ smtp_transport_options_block smtp_transport_option_defaults = { .tls_try_verify_hosts = US"*", .tls_verify_cert_hostnames = US"*", #endif +#ifdef SUPPORT_I18N + .utf8_downconvert = NULL, +#endif #ifndef DISABLE_DKIM .dkim = {.dkim_domain = NULL, @@ -2218,6 +2224,38 @@ sx->setting_up = FALSE; #ifdef SUPPORT_I18N if (sx->addrlist->prop.utf8_msg) { + uschar * s; + + /* If the transport sets a downconversion mode it overrides any set by ACL + for the message. */ + + if ((s = sx->ob->utf8_downconvert)) + { + if (!(s = expand_string(s))) + { + message = string_sprintf("failed to expand utf8_downconvert: %s", + expand_string_message); + set_errno_nohost(sx->addrlist, ERRNO_EXPANDFAIL, message, DEFER, FALSE); + yield = DEFER; + goto SEND_QUIT; + } + switch (*s) + { + case '1': sx->addrlist->prop.utf8_downcvt = TRUE; + sx->addrlist->prop.utf8_downcvt_maybe = FALSE; + break; + case '0': sx->addrlist->prop.utf8_downcvt = FALSE; + sx->addrlist->prop.utf8_downcvt_maybe = FALSE; + break; + case '-': if (s[1] == '1') + { + sx->addrlist->prop.utf8_downcvt = FALSE; + sx->addrlist->prop.utf8_downcvt_maybe = TRUE; + } + break; + } + } + sx->utf8_needed = !sx->addrlist->prop.utf8_downcvt && !sx->addrlist->prop.utf8_downcvt_maybe; DEBUG(D_transport) if (!sx->utf8_needed) @@ -2231,7 +2269,7 @@ if (sx->utf8_needed && !(sx->peer_offered & OPTION_UTF8)) errno = ERRNO_UTF8_FWD; goto RESPONSE_FAILED; } -#endif +#endif /*SUPPORT_I18N*/ #if defined(SUPPORT_TLS) && defined(EXPERIMENTAL_REQUIRETLS) /*XXX should tls_requiretls actually be per-addr? */ diff --git a/src/src/transports/smtp.h b/src/src/transports/smtp.h index 34c49d930..df9644377 100644 --- a/src/src/transports/smtp.h +++ b/src/src/transports/smtp.h @@ -88,6 +88,9 @@ typedef struct { uschar *tls_try_verify_hosts; uschar *tls_verify_cert_hostnames; #endif +#ifdef SUPPORT_I18N + uschar *utf8_downconvert; +#endif #ifndef DISABLE_DKIM struct ob_dkim dkim; #endif diff --git a/test/confs/4201 b/test/confs/4201 index 9242eb063..17afe9f78 100644 --- a/test/confs/4201 +++ b/test/confs/4201 @@ -124,5 +124,8 @@ local_delivery: rmt_smtp: driver = smtp +.ifdef STRICT + utf8_downconvert = STRICT +.endif # End diff --git a/test/log/4207 b/test/log/4207 index e39c0f5fb..3bd01a3dd 100644 --- a/test/log/4207 +++ b/test/log/4207 @@ -1,17 +1,23 @@ -1999-03-02 09:44:33 10HmaX-0005vi-00 <= 他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com U=CALLER P=utf8local-esmtp S=sss for user.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex -1999-03-02 09:44:33 10HmaX-0005vi-00 => xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@test.ex F= R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] C="250 OK id=10HmaY-0005vi-00" +1999-03-02 09:44:33 10HmaX-0005vi-00 <= 他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com U=CALLER P=utf8local-esmtp S=sss for user1.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex +1999-03-02 09:44:33 10HmaX-0005vi-00 => xn--user1.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex F= R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] C="250 OK id=10HmaY-0005vi-00" 1999-03-02 09:44:33 10HmaX-0005vi-00 Completed -1999-03-02 09:44:33 10HmaZ-0005vi-00 <= 他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com U=CALLER P=utf8local-esmtp S=sss for zuser.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex -1999-03-02 09:44:33 10HmaZ-0005vi-00 => xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@test.ex F= R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] C="250 OK id=10HmbA-0005vi-00" +1999-03-02 09:44:33 10HmaZ-0005vi-00 <= 他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com U=CALLER P=utf8local-esmtp S=sss for zuser2.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex +1999-03-02 09:44:33 10HmaZ-0005vi-00 => xn--user2.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex F= R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] C="250 OK id=10HmbA-0005vi-00" 1999-03-02 09:44:33 10HmaZ-0005vi-00 Completed +1999-03-02 09:44:33 10HmbB-0005vi-00 <= 他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com U=CALLER P=utf8local-esmtp S=sss for user3.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex +1999-03-02 09:44:33 10HmbB-0005vi-00 => xn--user3.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex F= R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] C="250 OK id=10HmbC-0005vi-00" +1999-03-02 09:44:33 10HmbB-0005vi-00 Completed ******** SERVER ******** 1999-03-02 09:44:33 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTP on port 1225 -1999-03-02 09:44:33 10HmaY-0005vi-00 <= xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com H=localhost (the.local.host.name) [127.0.0.1] P=esmtp S=sss id=E10HmaX-0005vi-00@the.local.host.name for xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@test.ex -1999-03-02 09:44:33 10HmbA-0005vi-00 <= xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com H=localhost (the.local.host.name) [127.0.0.1] P=esmtp S=sss id=E10HmaZ-0005vi-00@the.local.host.name for xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@test.ex +1999-03-02 09:44:33 10HmaY-0005vi-00 <= xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com H=localhost (the.local.host.name) [127.0.0.1] P=esmtp S=sss id=E10HmaX-0005vi-00@the.local.host.name for xn--user1.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex +1999-03-02 09:44:33 10HmbA-0005vi-00 <= xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com H=localhost (the.local.host.name) [127.0.0.1] P=esmtp S=sss id=E10HmaZ-0005vi-00@the.local.host.name for xn--user2.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex +1999-03-02 09:44:33 10HmbC-0005vi-00 <= xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com H=localhost (the.local.host.name) [127.0.0.1] P=esmtp S=sss id=E10HmbB-0005vi-00@the.local.host.name for xn--user3.-g95ww2bm7c8xj18gesgrby74dwqh18as2ft0ab05f5nc9w1befas47alnaxwd256esy4hea33e@test.ex 1999-03-02 09:44:33 Start queue run: pid=pppp -qqff -1999-03-02 09:44:33 10HmaY-0005vi-00 => :blackhole: R=localuser +1999-03-02 09:44:33 10HmaY-0005vi-00 => :blackhole: R=localuser 1999-03-02 09:44:33 10HmaY-0005vi-00 Completed -1999-03-02 09:44:33 10HmbA-0005vi-00 => :blackhole: R=localuser +1999-03-02 09:44:33 10HmbA-0005vi-00 => :blackhole: R=localuser 1999-03-02 09:44:33 10HmbA-0005vi-00 Completed +1999-03-02 09:44:33 10HmbC-0005vi-00 => :blackhole: R=localuser +1999-03-02 09:44:33 10HmbC-0005vi-00 Completed 1999-03-02 09:44:33 End queue run: pid=pppp -qqff diff --git a/test/runtest b/test/runtest index 9f556660d..a38f112af 100755 --- a/test/runtest +++ b/test/runtest @@ -1613,6 +1613,7 @@ $munges = |hosts_(avoid|nopass|noproxy|require|verify_avoid)_tls |socks_proxy |tls_[^ ]* + |utf8_downconvert )($|[ ]=)/x' }, diff --git a/test/scripts/0000-Basic/0577 b/test/scripts/0000-Basic/0577 index 1bd510ae3..bbd2954c7 100644 --- a/test/scripts/0000-Basic/0577 +++ b/test/scripts/0000-Basic/0577 @@ -1,2 +1,3 @@ # Test different variants of .includes +munge optional_config exim -bP config diff --git a/test/scripts/4200-International/4207 b/test/scripts/4200-International/4207 index 061a2c6b3..fa32bec92 100644 --- a/test/scripts/4200-International/4207 +++ b/test/scripts/4200-International/4207 @@ -9,7 +9,7 @@ exim -DSERVER=server -DOPTION="" -bd -oX PORT_D exim -bs -odi -DCONTROL="control=utf8_downconvert" EHLO client.bh MAIL FROM: <他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com> SMTPUTF8 -RCPT TO: +RCPT TO: DATA Subject: test @@ -22,7 +22,20 @@ QUIT exim -bs -odi -DCONTROL="control=utf8_downconvert/-1" EHLO client.bh MAIL FROM: <他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com> SMTPUTF8 -RCPT TO: +RCPT TO: +DATA +Subject: test + +body +. +QUIT +**** +# +# utf-8 from, never-downconvert set by ACL, overridden by transport +exim -bs -odi -DCONTROL="control=utf8_downconvert/0" -DSTRICT=1 +EHLO client.bh +MAIL FROM: <他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com> SMTPUTF8 +RCPT TO: DATA Subject: test diff --git a/test/stdout/0577 b/test/stdout/0577 index 80cf3da88..ef918a86b 100644 --- a/test/stdout/0577 +++ b/test/stdout/0577 @@ -14,7 +14,6 @@ gecos_name = CALLER_NAME dns_cname_loops = 9 chunking_advertise_hosts = # 1 "TESTSUITE/aux-var/std_conf_prefix" -tls_advertise_hosts = # 1 "TESTSUITE/test-config" # 2 "TESTSUITE/test-config" # 1 "TESTSUITE/confs/0577./aaa" diff --git a/test/stdout/4207 b/test/stdout/4207 index 948434ff3..34a2cfe36 100644 --- a/test/stdout/4207 +++ b/test/stdout/4207 @@ -22,3 +22,15 @@ 354 Enter message, ending with "." on a line by itself 250 OK id=10HmaZ-0005vi-00 221 the.local.host.name closing connection +220 the.local.host.name ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000 +250-the.local.host.name Hello CALLER at client.bh +250-SIZE 52428800 +250-8BITMIME +250-PIPELINING +250-SMTPUTF8 +250 HELP +250 OK +250 Accepted +354 Enter message, ending with "." on a line by itself +250 OK id=10HmbB-0005vi-00 +221 the.local.host.name closing connection -- cgit v1.2.3