diff options
author | Jeremy Harris <jgh146exb@wizmail.org> | 2015-04-20 16:48:36 +0100 |
---|---|---|
committer | Jeremy Harris <jgh146exb@wizmail.org> | 2015-04-21 15:53:32 +0100 |
commit | 3c8b357717e895d6dcddc7faa5b7a0eaf4c73417 (patch) | |
tree | c9b54008e61eed3caa7ed58063ec0d30e60b6874 | |
parent | e8f0fc219ceca2834ee2b6a1a1a9b960ead39a2b (diff) |
UTF8: MSA downconversions
-rw-r--r-- | doc/doc-txt/NewStuff | 5 | ||||
-rw-r--r-- | doc/doc-txt/experimental-spec.txt | 12 | ||||
-rw-r--r-- | src/src/acl.c | 156 | ||||
-rw-r--r-- | src/src/deliver.c | 15 | ||||
-rw-r--r-- | src/src/functions.h | 3 | ||||
-rw-r--r-- | src/src/globals.c | 1 | ||||
-rw-r--r-- | src/src/globals.h | 1 | ||||
-rw-r--r-- | src/src/routers/redirect.c | 4 | ||||
-rw-r--r-- | src/src/spool_in.c | 10 | ||||
-rw-r--r-- | src/src/spool_out.c | 9 | ||||
-rw-r--r-- | src/src/string.c | 46 | ||||
-rw-r--r-- | src/src/structs.h | 7 | ||||
-rw-r--r-- | src/src/transports/smtp.c | 74 | ||||
-rw-r--r-- | src/src/utf8.c | 27 | ||||
-rw-r--r-- | src/src/verify.c | 28 | ||||
-rw-r--r-- | test/confs/4201 | 2 | ||||
l--------- | test/confs/4207 | 1 | ||||
-rw-r--r-- | test/log/4207 | 9 | ||||
-rw-r--r-- | test/scripts/4200-International/4207 | 24 | ||||
-rw-r--r-- | test/stderr/4207 | 2 | ||||
-rw-r--r-- | test/stdout/4207 | 12 |
21 files changed, 347 insertions, 101 deletions
diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff index e4bc586a5..ef6a6dea9 100644 --- a/doc/doc-txt/NewStuff +++ b/doc/doc-txt/NewStuff @@ -23,7 +23,10 @@ Version 4.86 6. A commandline option to write a comment into the logfile. 7. If built with EXPERIMENTAL_SOCKS feature enabled, the smtp transport can - be configured to make connections via socks5 proxies + be configured to make connections via socks5 proxies. + + 8. If built with EXPERIMENTAL_INTERNATIONAL, support is included for + the transmission of UTF-8 envelope addresses. Version 4.85 diff --git a/doc/doc-txt/experimental-spec.txt b/doc/doc-txt/experimental-spec.txt index 819b47fd5..234bac0ab 100644 --- a/doc/doc-txt/experimental-spec.txt +++ b/doc/doc-txt/experimental-spec.txt @@ -1308,12 +1308,22 @@ New expansion operators: ${utf8_localpart_to_alabel:str} ${utf8_localpart_from_alabel:str} +New "control = utf8_downconvert" ACL modifier, +sets a flag requiring that addresses are converted to +a-label form before smtp delivery, for use in a +Message Submission Agent context. Can also be +phrased as "control = utf8_downconvert/1" and is +mandatory. The flag defaults to zero and can be cleared +by "control = utf8_downconvert/0". The value "-1" +may also be used, to use a-label for only if the +destination host does not support SMTPUTF8. + Known issues: - Currently LMTP is not supported. - DSN unitext handling is not present - no provision for converting logging from UTF-8 - VRFY and EXPN not handled - - MSA mode not handled (!) + - mua_wrapper not handled -------------------------------------------------------------- End of file diff --git a/src/src/acl.c b/src/src/acl.c index 3f513c328..c1402a0ff 100644 --- a/src/src/acl.c +++ b/src/src/acl.c @@ -181,17 +181,17 @@ that follows! */ enum { CONTROL_AUTH_UNADVERTISED, - #ifdef EXPERIMENTAL_BRIGHTMAIL +#ifdef EXPERIMENTAL_BRIGHTMAIL CONTROL_BMI_RUN, - #endif +#endif CONTROL_DEBUG, - #ifndef DISABLE_DKIM +#ifndef DISABLE_DKIM CONTROL_DKIM_VERIFY, - #endif - #ifdef EXPERIMENTAL_DMARC +#endif +#ifdef EXPERIMENTAL_DMARC CONTROL_DMARC_VERIFY, CONTROL_DMARC_FORENSIC, - #endif +#endif CONTROL_DSCP, CONTROL_ERROR, CONTROL_CASEFUL_LOCAL_PART, @@ -203,11 +203,14 @@ enum { CONTROL_QUEUE_ONLY, CONTROL_SUBMISSION, CONTROL_SUPPRESS_LOCAL_FIXUPS, - #ifdef WITH_CONTENT_SCAN +#ifdef WITH_CONTENT_SCAN CONTROL_NO_MBOX_UNSPOOL, - #endif +#endif CONTROL_FAKEDEFER, CONTROL_FAKEREJECT, +#ifdef EXPERIMENTAL_INTERNATIONAL + CONTROL_UTF8_DOWNCONVERT, +#endif CONTROL_NO_MULTILINE, CONTROL_NO_PIPELINING, CONTROL_NO_DELAY_FLUSH, @@ -221,17 +224,17 @@ and should be tidied up. */ static uschar *controls[] = { US"allow_auth_unadvertised", - #ifdef EXPERIMENTAL_BRIGHTMAIL +#ifdef EXPERIMENTAL_BRIGHTMAIL US"bmi_run", - #endif +#endif US"debug", - #ifndef DISABLE_DKIM +#ifndef DISABLE_DKIM US"dkim_disable_verify", - #endif - #ifdef EXPERIMENTAL_DMARC +#endif +#ifdef EXPERIMENTAL_DMARC US"dmarc_disable_verify", US"dmarc_enable_forensic", - #endif +#endif US"dscp", US"error", US"caseful_local_part", @@ -243,11 +246,14 @@ static uschar *controls[] = { US"queue_only", US"submission", US"suppress_local_fixups", - #ifdef WITH_CONTENT_SCAN +#ifdef WITH_CONTENT_SCAN US"no_mbox_unspool", - #endif +#endif US"fakedefer", US"fakereject", +#ifdef EXPERIMENTAL_INTERNATIONAL + US"utf8_downconvert", +#endif US"no_multiline_responses", US"no_pipelining", US"no_delay_flush", @@ -600,26 +606,26 @@ static unsigned int control_forbids[] = { (unsigned int) ~((1<<ACL_WHERE_CONNECT)|(1<<ACL_WHERE_HELO)), /* allow_auth_unadvertised */ - #ifdef EXPERIMENTAL_BRIGHTMAIL +#ifdef EXPERIMENTAL_BRIGHTMAIL 0, /* bmi_run */ - #endif +#endif 0, /* debug */ - #ifndef DISABLE_DKIM +#ifndef DISABLE_DKIM (1<<ACL_WHERE_DATA)|(1<<ACL_WHERE_NOTSMTP)| /* dkim_disable_verify */ - #ifndef DISABLE_PRDR +# ifndef DISABLE_PRDR (1<<ACL_WHERE_PRDR)| - #endif +# endif (1<<ACL_WHERE_NOTSMTP_START), - #endif +#endif - #ifdef EXPERIMENTAL_DMARC +#ifdef EXPERIMENTAL_DMARC (1<<ACL_WHERE_DATA)|(1<<ACL_WHERE_NOTSMTP)| /* dmarc_disable_verify */ (1<<ACL_WHERE_NOTSMTP_START), (1<<ACL_WHERE_DATA)|(1<<ACL_WHERE_NOTSMTP)| /* dmarc_enable_forensic */ (1<<ACL_WHERE_NOTSMTP_START), - #endif +#endif (1<<ACL_WHERE_NOTSMTP)| (1<<ACL_WHERE_NOTSMTP_START)| @@ -663,30 +669,34 @@ static unsigned int control_forbids[] = { (1<<ACL_WHERE_PREDATA)| (1<<ACL_WHERE_NOTSMTP_START)), - #ifdef WITH_CONTENT_SCAN +#ifdef WITH_CONTENT_SCAN (unsigned int) ~((1<<ACL_WHERE_MAIL)|(1<<ACL_WHERE_RCPT)| /* no_mbox_unspool */ (1<<ACL_WHERE_PREDATA)|(1<<ACL_WHERE_DATA)| // (1<<ACL_WHERE_PRDR)| /* Not allow one user to freeze for all */ (1<<ACL_WHERE_MIME)), - #endif +#endif (unsigned int) ~((1<<ACL_WHERE_MAIL)|(1<<ACL_WHERE_RCPT)| /* fakedefer */ (1<<ACL_WHERE_PREDATA)|(1<<ACL_WHERE_DATA)| - #ifndef DISABLE_PRDR +#ifndef DISABLE_PRDR (1<<ACL_WHERE_PRDR)| - #endif +#endif (1<<ACL_WHERE_MIME)), (unsigned int) ~((1<<ACL_WHERE_MAIL)|(1<<ACL_WHERE_RCPT)| /* fakereject */ (1<<ACL_WHERE_PREDATA)|(1<<ACL_WHERE_DATA)| - #ifndef DISABLE_PRDR +#ifndef DISABLE_PRDR (1<<ACL_WHERE_PRDR)| - #endif +#endif (1<<ACL_WHERE_MIME)), +#ifdef EXPERIMENTAL_INTERNATIONAL + 0, /* utf8_downconvert */ +#endif + (1<<ACL_WHERE_NOTSMTP)| /* no_multiline */ (1<<ACL_WHERE_NOTSMTP_START), @@ -709,37 +719,40 @@ typedef struct control_def { } control_def; static control_def controls_list[] = { - { US"allow_auth_unadvertised", CONTROL_AUTH_UNADVERTISED, FALSE }, + { US"allow_auth_unadvertised", CONTROL_AUTH_UNADVERTISED, FALSE }, #ifdef EXPERIMENTAL_BRIGHTMAIL - { US"bmi_run", CONTROL_BMI_RUN, FALSE }, + { US"bmi_run", CONTROL_BMI_RUN, FALSE }, #endif - { US"debug", CONTROL_DEBUG, TRUE }, + { US"debug", CONTROL_DEBUG, TRUE }, #ifndef DISABLE_DKIM - { US"dkim_disable_verify", CONTROL_DKIM_VERIFY, FALSE }, + { US"dkim_disable_verify", CONTROL_DKIM_VERIFY, FALSE }, #endif #ifdef EXPERIMENTAL_DMARC - { US"dmarc_disable_verify", CONTROL_DMARC_VERIFY, FALSE }, - { US"dmarc_enable_forensic", CONTROL_DMARC_FORENSIC, FALSE }, + { US"dmarc_disable_verify", CONTROL_DMARC_VERIFY, FALSE }, + { US"dmarc_enable_forensic", CONTROL_DMARC_FORENSIC, FALSE }, #endif - { US"dscp", CONTROL_DSCP, TRUE }, - { US"caseful_local_part", CONTROL_CASEFUL_LOCAL_PART, FALSE }, - { US"caselower_local_part", CONTROL_CASELOWER_LOCAL_PART, FALSE }, - { US"enforce_sync", CONTROL_ENFORCE_SYNC, FALSE }, - { US"freeze", CONTROL_FREEZE, TRUE }, - { US"no_callout_flush", CONTROL_NO_CALLOUT_FLUSH, FALSE }, - { US"no_delay_flush", CONTROL_NO_DELAY_FLUSH, FALSE }, - { US"no_enforce_sync", CONTROL_NO_ENFORCE_SYNC, FALSE }, - { US"no_multiline_responses", CONTROL_NO_MULTILINE, FALSE }, - { US"no_pipelining", CONTROL_NO_PIPELINING, FALSE }, - { US"queue_only", CONTROL_QUEUE_ONLY, FALSE }, + { US"dscp", CONTROL_DSCP, TRUE }, + { US"caseful_local_part", CONTROL_CASEFUL_LOCAL_PART, FALSE }, + { US"caselower_local_part", CONTROL_CASELOWER_LOCAL_PART, FALSE }, + { US"enforce_sync", CONTROL_ENFORCE_SYNC, FALSE }, + { US"freeze", CONTROL_FREEZE, TRUE }, + { US"no_callout_flush", CONTROL_NO_CALLOUT_FLUSH, FALSE }, + { US"no_delay_flush", CONTROL_NO_DELAY_FLUSH, FALSE }, + { US"no_enforce_sync", CONTROL_NO_ENFORCE_SYNC, FALSE }, + { US"no_multiline_responses", CONTROL_NO_MULTILINE, FALSE }, + { US"no_pipelining", CONTROL_NO_PIPELINING, FALSE }, + { US"queue_only", CONTROL_QUEUE_ONLY, FALSE }, #ifdef WITH_CONTENT_SCAN - { US"no_mbox_unspool", CONTROL_NO_MBOX_UNSPOOL, FALSE }, + { US"no_mbox_unspool", CONTROL_NO_MBOX_UNSPOOL, FALSE }, #endif - { US"fakedefer", CONTROL_FAKEDEFER, TRUE }, - { US"fakereject", CONTROL_FAKEREJECT, TRUE }, - { US"submission", CONTROL_SUBMISSION, TRUE }, + { US"fakedefer", CONTROL_FAKEDEFER, TRUE }, + { US"fakereject", CONTROL_FAKEREJECT, TRUE }, + { US"submission", CONTROL_SUBMISSION, TRUE }, { US"suppress_local_fixups", CONTROL_SUPPRESS_LOCAL_FIXUPS, FALSE }, - { US"cutthrough_delivery", CONTROL_CUTTHROUGH_DELIVERY, FALSE } + { US"cutthrough_delivery", CONTROL_CUTTHROUGH_DELIVERY, FALSE }, +#ifdef EXPERIMENTAL_INTERNATIONAL + { US"utf8_downconvert", CONTROL_UTF8_DOWNCONVERT, TRUE } +#endif }; /* Support data structures for Client SMTP Authorization. acl_verify_csa() @@ -2080,7 +2093,11 @@ else if (verify_sender_address != NULL) sender_vaddr = deliver_make_addr(verify_sender_address, TRUE); #ifdef EXPERIMENTAL_INTERNATIONAL - sender_vaddr->prop.utf8 = message_smtputf8; + if ((sender_vaddr->prop.utf8_msg = message_smtputf8)) + { + sender_vaddr->prop.utf8_downcvt = message_utf8_downconvert == 1; + sender_vaddr->prop.utf8_downcvt_maybe = message_utf8_downconvert == -1; + } #endif if (no_details) setflag(sender_vaddr, af_sverify_told); if (verify_sender_address[0] != 0) @@ -3377,6 +3394,24 @@ for (; cb != NULL; cb = cb->next) arg, *log_msgptr); } return ERROR; + + #ifdef EXPERIMENTAL_INTERNATIONAL + case CONTROL_UTF8_DOWNCONVERT: + if (*p == '/') + { + if (p[1] == '1') { message_utf8_downconvert = 1; p += 2; break; } + if (p[1] == '0') { message_utf8_downconvert = 0; p += 2; break; } + if (p[1] == '-' && p[2] == '1') + { message_utf8_downconvert = -1; p += 3; break; } + *log_msgptr = US"bad option value for control=utf8_downconvert"; + } + else + { + message_utf8_downconvert = 1; break; + } + return ERROR; + #endif + } break; } @@ -3390,14 +3425,9 @@ for (; cb != NULL; cb = cb->next) /* Run the dcc backend. */ rc = dcc_process(&ss); /* Modify return code based upon the existance of options. */ - while ((ss = string_nextinlist(&list, &sep, big_buffer, big_buffer_size)) - != NULL) { + while ((ss = string_nextinlist(&list, &sep, big_buffer, big_buffer_size))) if (strcmpic(ss, US"defer_ok") == 0 && rc == DEFER) - { - /* FAIL so that the message is passed to the next ACL */ - rc = FAIL; - } - } + rc = FAIL; /* FAIL so that the message is passed to the next ACL */ } break; #endif @@ -4382,7 +4412,11 @@ if (where == ACL_WHERE_RCPT) return DEFER; } #ifdef EXPERIMENTAL_INTERNATIONAL - addr->prop.utf8 = message_smtputf8; + if ((addr->prop.utf8_msg = message_smtputf8)) + { + addr->prop.utf8_downcvt = message_utf8_downconvert == 1; + addr->prop.utf8_downcvt_maybe = message_utf8_downconvert == -1; + } #endif deliver_domain = addr->domain; deliver_localpart = addr->local_part; diff --git a/src/src/deliver.c b/src/src/deliver.c index f208f3edd..58b9d3a01 100644 --- a/src/src/deliver.c +++ b/src/src/deliver.c @@ -822,7 +822,14 @@ if (log_extra_selector & LX_incoming_interface && sending_ip_address) /* for the port: string_sprintf("%d", sending_port) */ if ((log_extra_selector & LX_sender_on_delivery) != 0 || msg) - s = string_append(s, &size, &ptr, 3, US" F=<", sender_address, US">"); + s = string_append(s, &size, &ptr, 3, US" F=<", +#ifdef EXPERIMENTAL_INTERNATIONAL + testflag(addr, af_utf8_downcvt) + ? string_address_utf8_to_alabel(sender_address, NULL) + : +#endif + sender_address, + US">"); #ifdef EXPERIMENTAL_SRS if(addr->prop.srs_sender) @@ -5595,7 +5602,11 @@ if (process_recipients != RECIP_IGNORE) address_item *new = deliver_make_addr(r->address, FALSE); new->prop.errors_address = r->errors_to; #ifdef EXPERIMENTAL_INTERNATIONAL - new->prop.utf8 = message_smtputf8; + if ((new->prop.utf8_msg = message_smtputf8)) + { + new->prop.utf8_downcvt = message_utf8_downconvert == 1; + new->prop.utf8_downcvt_maybe = message_utf8_downconvert == -1; + } DEBUG(D_deliver) if (message_smtputf8) debug_printf("utf8\n"); #endif diff --git a/src/src/functions.h b/src/src/functions.h index d1ada3844..1708b7a07 100644 --- a/src/src/functions.h +++ b/src/src/functions.h @@ -422,8 +422,7 @@ extern const uschar *string_printing2(const uschar *, BOOL); extern uschar *string_split_message(uschar *); extern uschar *string_unprinting(uschar *); #ifdef EXPERIMENTAL_INTERNATIONAL -extern uschar *string_address_alabel_to_utf8(const uschar *, uschar **); -extern uschar *string_address_utf8_to_alabel(uschar *, uschar **, int *); +extern uschar *string_address_utf8_to_alabel(const uschar *, uschar **); extern uschar *string_domain_alabel_to_utf8(const uschar *, uschar **); extern uschar *string_domain_utf8_to_alabel(const uschar *, uschar **); extern uschar *string_localpart_alabel_to_utf8(const uschar *, uschar **); diff --git a/src/src/globals.c b/src/src/globals.c index 2bf4d0a02..a71c80ed9 100644 --- a/src/src/globals.c +++ b/src/src/globals.c @@ -915,6 +915,7 @@ int message_size = 0; uschar *message_size_limit = US"50M"; #ifdef EXPERIMENTAL_INTERNATIONAL BOOL message_smtputf8 = FALSE; +int message_utf8_downconvert = 0; /* -1 ifneeded; 0 never; 1 always */ #endif uschar message_subdir[2] = { 0, 0 }; uschar *message_reference = NULL; diff --git a/src/src/globals.h b/src/src/globals.h index 7cbf7bfab..8aa69bff6 100644 --- a/src/src/globals.h +++ b/src/src/globals.h @@ -572,6 +572,7 @@ extern int message_size; /* Size of message */ extern uschar *message_size_limit; /* As it says */ #ifdef EXPERIMENTAL_INTERNATIONAL extern BOOL message_smtputf8; /* Internationalized mail handling */ +extern int message_utf8_downconvert; /* convert from utf8 */ const extern pcre *regex_UTF8; /* For recognizing SMTPUTF8 settings */ #endif extern uschar message_subdir[]; /* Subdirectory for messages */ diff --git a/src/src/routers/redirect.c b/src/src/routers/redirect.c index 8f1c2c3d2..9e57c8be6 100644 --- a/src/src/routers/redirect.c +++ b/src/src/routers/redirect.c @@ -452,7 +452,7 @@ while (generated != NULL) } #ifdef EXPERIMENTAL_INTERNATIONAL - next->prop.utf8 = string_is_utf8(next->address) + next->prop.utf8_msg = string_is_utf8(next->address) || (sender_address && string_is_utf8(sender_address)); #endif @@ -476,7 +476,7 @@ while (generated != NULL) debug_printf("gid=unset "); #ifdef EXPERIMENTAL_INTERNATIONAL - if (next->prop.utf8) debug_printf("utf8 "); + if (next->prop.utf8_msg) debug_printf("utf8 "); #endif debug_printf("home=%s\n", next->home_dir); diff --git a/src/src/spool_in.c b/src/src/spool_in.c index 742f4b579..558d955c1 100644 --- a/src/src/spool_in.c +++ b/src/src/spool_in.c @@ -301,6 +301,7 @@ spam_score_int = NULL; #if defined(EXPERIMENTAL_INTERNATIONAL) && !defined(COMPILE_UTILITY) message_smtputf8 = FALSE; +message_utf8_downconvert = 0; #endif dsn_ret = 0; @@ -600,6 +601,15 @@ for (;;) break; #endif +#if defined(EXPERIMENTAL_INTERNATIONAL) && !defined(COMPILE_UTILITY) + case 'u': + if (Ustrncmp(p, "tf8_downcvt", 11) == 0) + message_utf8_downconvert = 1; + else if (Ustrncmp(p, "tf8_downcvt_opt", 15) == 0) + message_utf8_downconvert = -1; + break; +#endif + default: /* Present because some compilers complain if all */ break; /* possibilities are not covered. */ } diff --git a/src/src/spool_out.c b/src/src/spool_out.c index 6d22bff2c..48f27a8c6 100644 --- a/src/src/spool_out.c +++ b/src/src/spool_out.c @@ -246,7 +246,12 @@ if (tls_in.ocsp) fprintf(f, "-tls_ocsp %d\n", tls_in.ocsp); #endif #ifdef EXPERIMENTAL_INTERNATIONAL -if (message_smtputf8) fprintf(f, "-smtputf8\n"); +if (message_smtputf8) + { + fprintf(f, "-smtputf8\n"); + if (message_utf8_downconvert) + fprintf(f, "-utf8_downcvt%s\n", message_utf8_downconvert < 0 ? "_opt" : ""); + } #endif /* Write the dsn flags to the spool header file */ @@ -508,3 +513,5 @@ return TRUE; #endif /* End of spool_out.c */ +/* vi: aw ai sw=2 +*/ diff --git a/src/src/string.c b/src/src/string.c index e169a9f05..c50a347c2 100644 --- a/src/src/string.c +++ b/src/src/string.c @@ -1547,14 +1547,35 @@ static uschar * string_get_localpart(address_item *addr, uschar *yield, int *sizeptr, int *ptrptr) { -if (testflag(addr, af_include_affixes) && addr->prefix != NULL) - yield = string_cat(yield, sizeptr, ptrptr, addr->prefix, - Ustrlen(addr->prefix)); -yield = string_cat(yield, sizeptr, ptrptr, addr->local_part, - Ustrlen(addr->local_part)); -if (testflag(addr, af_include_affixes) && addr->suffix != NULL) - yield = string_cat(yield, sizeptr, ptrptr, addr->suffix, - Ustrlen(addr->suffix)); +uschar * s; + +s = addr->prefix; +if (testflag(addr, af_include_affixes) && s) + { +#ifdef EXPERIMENTAL_INTERNATIONAL + if (testflag(addr, af_utf8_downcvt)) + s = string_localpart_utf8_to_alabel(s, NULL); +#endif + yield = string_cat(yield, sizeptr, ptrptr, s, Ustrlen(s)); + } + +s = addr->local_part; +#ifdef EXPERIMENTAL_INTERNATIONAL +if (testflag(addr, af_utf8_downcvt)) + s = string_localpart_utf8_to_alabel(s, NULL); +#endif +yield = string_cat(yield, sizeptr, ptrptr, s, Ustrlen(s)); + +s = addr->suffix; +if (testflag(addr, af_include_affixes) && s) + { +#ifdef EXPERIMENTAL_INTERNATIONAL + if (testflag(addr, af_utf8_downcvt)) + s = string_localpart_utf8_to_alabel(s, NULL); +#endif + yield = string_cat(yield, sizeptr, ptrptr, s, Ustrlen(s)); + } + return yield; } @@ -1615,10 +1636,15 @@ else { if (addr->local_part != NULL) { + const uschar * s; yield = string_get_localpart(addr, yield, &size, &ptr); yield = string_cat(yield, &size, &ptr, US"@", 1); - yield = string_cat(yield, &size, &ptr, addr->domain, - Ustrlen(addr->domain) ); + s = addr->domain; +#ifdef EXPERIMENTAL_INTERNATIONAL + if (testflag(addr, af_utf8_downcvt)) + s = string_localpart_utf8_to_alabel(s, NULL); +#endif + yield = string_cat(yield, &size, &ptr, s, Ustrlen(s) ); } else { diff --git a/src/src/structs.h b/src/src/structs.h index 99d65cfae..c181f3f6e 100644 --- a/src/src/structs.h +++ b/src/src/structs.h @@ -460,7 +460,9 @@ typedef struct address_item_propagated { uschar *srs_sender; /* Change return path when delivering */ #endif #ifdef EXPERIMENTAL_INTERNATIONAL - BOOL utf8; /* requires SMTPUTF8 processing */ + BOOL utf8_msg:1; /* requires SMTPUTF8 processing */ + BOOL utf8_downcvt:1; /* mandatory downconvert on delivery */ + BOOL utf8_downcvt_maybe:1; /* optional downconvert on delivery */ #endif } address_item_propagated; @@ -500,6 +502,9 @@ typedef struct address_item_propagated { #ifdef EXPERIMENTAL_DANE # define af_dane_verified 0x20000000 /* TLS cert verify done with DANE */ #endif +#ifdef EXPERIMENTAL_INTERNATIONAL +# define af_utf8_downcvt 0x40000000 /* downconvert was done for delivery */ +#endif /* These flags must be propagated when a child is created */ diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c index 65bb1de22..7537e6e4b 100644 --- a/src/src/transports/smtp.c +++ b/src/src/transports/smtp.c @@ -1366,6 +1366,7 @@ BOOL prdr_offered = FALSE; BOOL prdr_active; #endif #ifdef EXPERIMENTAL_INTERNATIONAL +BOOL utf8_needed = FALSE; BOOL utf8_offered = FALSE; #endif BOOL dsn_all_lasthop = TRUE; @@ -1642,10 +1643,17 @@ goto SEND_QUIT; #endif #ifdef EXPERIMENTAL_INTERNATIONAL - utf8_offered = esmtp - && addrlist->prop.utf8 - && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0, - PCRE_EOPT, NULL, 0) >= 0; + if (addrlist->prop.utf8_msg) + { + utf8_needed = !addrlist->prop.utf8_downcvt + && !addrlist->prop.utf8_downcvt_maybe; + DEBUG(D_transport) if (!utf8_needed) debug_printf("utf8: %s downconvert\n", + addrlist->prop.utf8_downcvt ? "mandatory" : "optional"); + + utf8_offered = esmtp + && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0, + PCRE_EOPT, NULL, 0) >= 0; + } #endif } @@ -1862,10 +1870,10 @@ if (continue_hostname == NULL #endif #ifdef EXPERIMENTAL_INTERNATIONAL - utf8_offered = esmtp - && addrlist->prop.utf8 - && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0, - PCRE_EOPT, NULL, 0) >= 0; + if (addrlist->prop.utf8_msg) + utf8_offered = esmtp + && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0, + PCRE_EOPT, NULL, 0) >= 0; #endif /* Note if the server supports DSN */ @@ -1896,7 +1904,7 @@ setting_up = FALSE; #ifdef EXPERIMENTAL_INTERNATIONAL /* If this is an international message we need the host to speak SMTPUTF8 */ -if (addrlist->prop.utf8 && !utf8_offered) +if (utf8_needed && !utf8_offered) { errno = ERRNO_UTF8_FWD; goto RESPONSE_FAILED; @@ -1980,7 +1988,7 @@ if (prdr_offered) #endif #ifdef EXPERIMENTAL_INTERNATIONAL -if (addrlist->prop.utf8) +if (addrlist->prop.utf8_msg && !addrlist->prop.utf8_downcvt && utf8_offered) sprintf(CS p, " SMTPUTF8"), p += 9; #endif @@ -2037,8 +2045,31 @@ buffer. */ pending_MAIL = TRUE; /* The block starts with MAIL */ -rc = smtp_write_command(&outblock, smtp_use_pipelining, - "MAIL FROM:<%s>%s\r\n", return_path, buffer); + { + uschar * s = return_path; +#ifdef EXPERIMENTAL_INTERNATIONAL + uschar * errstr = NULL; + + /* If we must downconvert, do the from-address here. Remember we had to + for the to-addresses (done below), and also (ugly) for re-doing when building + the delivery log line. */ + + if (addrlist->prop.utf8_msg && (addrlist->prop.utf8_downcvt || !utf8_offered)) + { + if (s = string_address_utf8_to_alabel(return_path, &errstr), errstr) + { + set_errno(addrlist, ERRNO_EXPANDFAIL, errstr, DEFER, FALSE, NULL); + yield = ERROR; + goto SEND_QUIT; + } + setflag(addrlist, af_utf8_downcvt); + } +#endif + + rc = smtp_write_command(&outblock, smtp_use_pipelining, + "MAIL FROM:<%s>%s\r\n", s, buffer); + } + mail_command = string_copy(big_buffer); /* Save for later error message */ switch(rc) @@ -2080,6 +2111,7 @@ for (addr = first_addr; { int count; BOOL no_flush; + uschar * rcpt_addr; addr->dsn_aware = smtp_use_dsn ? dsn_support_yes : dsn_support_no; @@ -2124,8 +2156,24 @@ for (addr = first_addr; yield as OK, because this error can often mean that there is a problem with just one address, so we don't want to delay the host. */ + rcpt_addr = transport_rcpt_address(addr, tblock->rcpt_include_affixes); + +#ifdef EXPERIMENTAL_INTERNATIONAL + { + uschar * dummy_errstr; + if ( testflag(addrlist, af_utf8_downcvt) + && (rcpt_addr = string_address_utf8_to_alabel(rcpt_addr, &dummy_errstr), + dummy_errstr + ) ) + { + errno = ERRNO_EXPANDFAIL; + goto SEND_FAILED; + } + } +#endif + count = smtp_write_command(&outblock, no_flush, "RCPT TO:<%s>%s%s\r\n", - transport_rcpt_address(addr, tblock->rcpt_include_affixes), igquotstr, buffer); + rcpt_addr, igquotstr, buffer); if (count < 0) goto SEND_FAILED; if (count > 0) diff --git a/src/src/utf8.c b/src/src/utf8.c index 6bc0c2ed5..09ebdf128 100644 --- a/src/src/utf8.c +++ b/src/src/utf8.c @@ -24,6 +24,7 @@ return FALSE; /**************************************************/ /* Domain conversions */ +/* the *err string pointer should be null before the call */ uschar * string_domain_utf8_to_alabel(const uschar * utf8, uschar ** err) @@ -68,6 +69,7 @@ return s; /**************************************************/ /* localpart conversions */ +/* the *err string pointer should be null before the call */ uschar * @@ -126,6 +128,31 @@ return res; } +/**************************************************/ +/* whole address conversion */ +/* the *err string pointer should be null before the call */ + +uschar * +string_address_utf8_to_alabel(const uschar * utf8, uschar ** err) +{ +const uschar * s; +uschar * l; +uschar * d; + +for (s = utf8; *s; s++) + if (*s == '@') + { + l = string_copyn(utf8, s - utf8); + return (l = string_localpart_utf8_to_alabel(l, err), err && *err) + || (d = string_domain_utf8_to_alabel(++s, err), err && *err) + ? NULL + : string_sprintf("%s@%s", l, d); + } +return string_localpart_utf8_to_alabel(utf8, err); +} + + + /************************************************* * Report the library versions. * *************************************************/ diff --git a/src/src/verify.c b/src/src/verify.c index 4e9b563fa..28013fa35 100644 --- a/src/src/verify.c +++ b/src/src/verify.c @@ -173,6 +173,9 @@ dbdata_callout_cache new_domain_record; dbdata_callout_cache_address new_address_record; host_item *host; time_t callout_start_time; +#ifdef EXPERIMENTAL_INTERNATIONAL +BOOL utf8_offered = FALSE; +#endif new_domain_record.result = ccache_unknown; new_domain_record.postmaster_result = ccache_unknown; @@ -921,22 +924,35 @@ can do it there for the non-rcpt-verify case. For this we keep an addresscount. } #ifdef EXPERIMENTAL_INTERNATIONAL - else if ( addr->prop.utf8 + else if ( addr->prop.utf8_msg + && !addr->prop.utf8_downcvt && !( esmtp && ( regex_UTF8 || ( (regex_UTF8 = regex_must_compile( US"\\n250[\\s\\-]SMTPUTF8(\\s|\\n|$)", FALSE, TRUE)), TRUE ) ) - && pcre_exec(regex_UTF8, NULL, CS responsebuffer, - Ustrlen(responsebuffer), 0, PCRE_EOPT, NULL, 0) >= 0 - ) ) + && ( (utf8_offered = pcre_exec(regex_UTF8, NULL, + CS responsebuffer, Ustrlen(responsebuffer), + 0, PCRE_EOPT, NULL, 0) >= 0) + || addr->prop.utf8_downcvt_maybe + ) ) ) { HDEBUG(D_acl|D_v) debug_printf("utf8 required but not offered\n"); errno = ERRNO_UTF8_FWD; setflag(addr, af_verify_nsfail); done = FALSE; } + else if ( addr->prop.utf8_msg + && (addr->prop.utf8_downcvt || !utf8_offered) + && (from_address = string_address_utf8_to_alabel(from_address, + &addr->message), addr->message) + ) + { + errno = ERRNO_EXPANDFAIL; + setflag(addr, af_verify_nsfail); + done = FALSE; + } #endif /* If we haven't authenticated, but are required to, give up. */ @@ -958,7 +974,7 @@ can do it there for the non-rcpt-verify case. For this we keep an addresscount. /* Send the MAIL command */ (smtp_write_command(&outblock, FALSE, #ifdef EXPERIMENTAL_INTERNATIONAL - addr->prop.utf8 + addr->prop.utf8_msg ? "MAIL FROM:<%s>%s SMTPUTF8\r\n" : #endif @@ -1049,7 +1065,7 @@ can do it there for the non-rcpt-verify case. For this we keep an addresscount. smtp_write_command(&outblock, FALSE, #ifdef EXPERIMENTAL_INTERNATIONAL - addr->prop.utf8 + addr->prop.utf8_msg ? "MAIL FROM:<%s> SMTPUTF8\r\n" : #endif diff --git a/test/confs/4201 b/test/confs/4201 index b34c7c1f9..b1fb7a6ee 100644 --- a/test/confs/4201 +++ b/test/confs/4201 @@ -39,7 +39,7 @@ begin acl check_recipient: accept hosts = : accept domains = +local_domains - local_parts = ^user.*\$ + local_parts = ^(xn--)?user.*\$ deny message = relay not permitted .else diff --git a/test/confs/4207 b/test/confs/4207 new file mode 120000 index 000000000..73a348fee --- /dev/null +++ b/test/confs/4207 @@ -0,0 +1 @@ +4201
\ No newline at end of file diff --git a/test/log/4207 b/test/log/4207 new file mode 100644 index 000000000..e1dd4f732 --- /dev/null +++ b/test/log/4207 @@ -0,0 +1,9 @@ +1999-03-02 09:44:33 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTP on port 1225 +1999-03-02 09:44:33 10HmaX-0005vi-00 <= 他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com U=CALLER P=utf8local-esmtp S=sss for user.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex +1999-03-02 09:44:33 10HmaY-0005vi-00 <= xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com H=localhost (the.local.host.name) [127.0.0.1] P=esmtp S=sss id=E10HmaX-0005vi-00@the.local.host.name for xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@test.ex +1999-03-02 09:44:33 10HmaX-0005vi-00 => xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@xn--test.ex- <user.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex> F=<xn--ihqwcrb4cv8a8dqg056pqjye@hebrew.xn--4dbcagdahymbxekheh6e0a7fei0b.com> R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] C="250 OK id=10HmaY-0005vi-00" +1999-03-02 09:44:33 10HmaX-0005vi-00 Completed +1999-03-02 09:44:33 Start queue run: pid=pppp -qqff +1999-03-02 09:44:33 10HmaY-0005vi-00 => :blackhole: <xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@test.ex> R=localuser +1999-03-02 09:44:33 10HmaY-0005vi-00 Completed +1999-03-02 09:44:33 End queue run: pid=pppp -qqff diff --git a/test/scripts/4200-International/4207 b/test/scripts/4200-International/4207 new file mode 100644 index 000000000..66e77f0a9 --- /dev/null +++ b/test/scripts/4200-International/4207 @@ -0,0 +1,24 @@ +# Internationalised mail: control = utf8_downconvert +# Exim test configuration 4207 +# +exim -DSERVER=server -bd -oX PORT_D +**** +# +# +# utf-8 from, mandatory downconvert +exim -bs -odi -DCONTROL="control=utf8_downconvert" +EHLO client.bh +MAIL FROM: <他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com> SMTPUTF8 +RCPT TO: <user.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex> +DATA +Subject: test + +body +. +QUIT +**** +# +killdaemon +exim -DSERVER=server -qqff +**** +no_msglog_check diff --git a/test/stderr/4207 b/test/stderr/4207 new file mode 100644 index 000000000..045fadc9b --- /dev/null +++ b/test/stderr/4207 @@ -0,0 +1,2 @@ + +******** SERVER ******** diff --git a/test/stdout/4207 b/test/stdout/4207 new file mode 100644 index 000000000..94a4e18d7 --- /dev/null +++ b/test/stdout/4207 @@ -0,0 +1,12 @@ +220 the.local.host.name ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000
+250-the.local.host.name Hello CALLER at client.bh
+250-SIZE 52428800
+250-8BITMIME
+250-PIPELINING
+250-SMTPUTF8
+250 HELP
+250 OK
+250 Accepted
+354 Enter message, ending with "." on a line by itself
+250 OK id=10HmaX-0005vi-00
+221 the.local.host.name closing connection
|